Found 298 jobs Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f4db4293420] , 2: gsignal() Ignoring malparsed backtrace: 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:35:43.121 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:35:43.122 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:35:43.122 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:36:30.130 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:36:30.130 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:36:30.131 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:38:38.470 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:38:38.470 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:38:38.471 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:39:59.778 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:39:59.778 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:39:59.779 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:40:46.811 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:40:46.812 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:40:46.813 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:42:08.118 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:08.118 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:08.119 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:42:55.129 INFO:tasks.ceph:pgid 3.6 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:55.129 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:55.130 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. Ignoring malparsed backtrace: 2023-03-29T23:44:16.449 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:44:16.449 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:44:16.450 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:45:23.759 INFO:teuthology.orchestra.run.smithi072.stdout: , dumped all , 2023-03-29T23:45:23.773 INFO:teuthology.orchestra.run.smithi072.stdout:{"pg_ready":true,"pg_map":{"version":1294,"stamp":"2023-03-29T23:45:22.955407+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":534,"num_write_kb":8435,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":494,"ondisk_log_size":494,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47240,"kb_used_data":15128,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098360,"statfs":{"total":644245094400,"available":644196720640,"internally_reserved":0,"allocated":15491072,"data_stored":14573894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003770"},"pg_stats":[{"pgid":"3.b","version":"19'26","reported_seq":36,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:06.497117+0000","last_change":"2023-03-29T23:02:55.829125+0000","last_active":"2023-03-29T23:03:06.497117+0000","last_peered":"2023-03-29T23:03:06.497117+0000","last_clean":"2023-03-29T23:03:06.497117+0000","last_became_active":"2023-03-29T23:02:55.828791+0000","last_became_peered":"2023-03-29T23:02:55.828791+0000","last_unstale":"2023-03-29T23:03:06.497117+0000","last_undegraded":"2023-03-29T23:03:06.497117+0000","last_fullsized":"2023-03-29T23:03:06.497117+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:14:34.429204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.828630+0000","last_change":"2023-03-29T23:02:55.833364+0000","last_active":"2023-03-29T23:02:56.828630+0000","last_peered":"2023-03-29T23:02:56.828630+0000","last_clean":"2023-03-29T23:02:56.828630+0000","last_became_active":"2023-03-29T23:02:55.833083+0000","last_became_peered":"2023-03-29T23:02:55.833083+0000","last_unstale":"2023-03-29T23:02:56.828630+0000","last_undegraded":"2023-03-29T23:02:56.828630+0000","last_fullsized":"2023-03-29T23:02:56.828630+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T09:30:35.161625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"19'64","reported_seq":74,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:11.546610+0000","last_change":"2023-03-29T23:02:55.834770+0000","last_active":"2023-03-29T23:03:11.546610+0000","last_peered":"2023-03-29T23:03:11.546610+0000","last_clean":"2023-03-29T23:03:11.546610+0000","last_became_active":"2023-03-29T23:02:55.834517+0000","last_became_peered":"2023-03-29T23:02:55.834517+0000","last_unstale":"2023-03-29T23:03:11.546610+0000","last_undegraded":"2023-03-29T23:03:11.546610+0000","last_fullsized":"2023-03-29T23:03:11.546610+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:14:34.429204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:57.623892+0000","last_change":"2023-03-29T23:02:55.832500+0000","last_active":"2023-03-29T23:02:57.623892+0000","last_peered":"2023-03-29T23:02:57.623892+0000","last_clean":"2023-03-29T23:02:57.623892+0000","last_became_active":"2023-03-29T23:02:55.832252+0000","last_became_peered":"2023-03-29T23:02:55.832252+0000","last_unstale":"2023-03-29T23:02:57.623892+0000","last_undegraded":"2023-03-29T23:02:57.623892+0000","last_fullsized":"2023-03-29T23:02:57.623892+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T09:30:35.161625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"14'97","reported_seq":141,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827817+0000","last_change":"2023-03-29T23:02:45.772706+0000","last_active":"2023-03-29T23:02:55.827817+0000","last_peered":"2023-03-29T23:02:55.827817+0000","last_clean":"2023-03-29T23:02:55.827817+0000","last_became_active":"2023-03-29T23:02:45.772124+0000","last_became_peered":"2023-03-29T23:02:45.772124+0000","last_unstale":"2023-03-29T23:02:55.827817+0000","last_undegraded":"2023-03-29T23:02:55.827817+0000","last_fullsized":"2023-03-29T23:02:55.827817+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:08:58.500583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.822203+0000","last_change":"2023-03-29T23:02:45.798160+0000","last_active":"2023-03-29T23:02:55.822203+0000","last_peered":"2023-03-29T23:02:55.822203+0000","last_clean":"2023-03-29T23:02:55.822203+0000","last_became_active":"2023-03-29T23:02:45.797639+0000","last_became_peered":"2023-03-29T23:02:45.797639+0000","last_unstale":"2023-03-29T23:02:55.822203+0000","last_undegraded":"2023-03-29T23:02:55.822203+0000","last_fullsized":"2023-03-29T23:02:55.822203+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T11:02:14.658224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"19'30","reported_seq":40,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:05.052479+0000","last_change":"2023-03-29T23:02:55.831017+0000","last_active":"2023-03-29T23:03:05.052479+0000","last_peered":"2023-03-29T23:03:05.052479+0000","last_clean":"2023-03-29T23:03:05.052479+0000","last_became_active":"2023-03-29T23:02:55.830688+0000","last_became_peered":"2023-03-29T23:02:55.830688+0000","last_unstale":"2023-03-29T23:03:05.052479+0000","last_undegraded":"2023-03-29T23:03:05.052479+0000","last_fullsized":"2023-03-29T23:03:05.052479+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:15:50.681015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.822940+0000","last_change":"2023-03-29T23:02:45.738317+0000","last_active":"2023-03-29T23:02:55.822940+0000","last_peered":"2023-03-29T23:02:55.822940+0000","last_clean":"2023-03-29T23:02:55.822940+0000","last_became_active":"2023-03-29T23:02:45.737938+0000","last_became_peered":"2023-03-29T23:02:45.737938+0000","last_unstale":"2023-03-29T23:02:55.822940+0000","last_undegraded":"2023-03-29T23:02:55.822940+0000","last_fullsized":"2023-03-29T23:02:55.822940+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:07:10.271259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"19'34","reported_seq":44,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:13.432024+0000","last_change":"2023-03-29T23:02:55.830034+0000","last_active":"2023-03-29T23:03:13.432024+0000","last_peered":"2023-03-29T23:03:13.432024+0000","last_clean":"2023-03-29T23:03:13.432024+0000","last_became_active":"2023-03-29T23:02:55.829749+0000","last_became_peered":"2023-03-29T23:02:55.829749+0000","last_unstale":"2023-03-29T23:03:13.432024+0000","last_undegraded":"2023-03-29T23:03:13.432024+0000","last_fullsized":"2023-03-29T23:03:13.432024+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:48:37.198336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.831154+0000","last_change":"2023-03-29T23:02:45.739333+0000","last_active":"2023-03-29T23:02:55.831154+0000","last_peered":"2023-03-29T23:02:55.831154+0000","last_clean":"2023-03-29T23:02:55.831154+0000","last_became_active":"2023-03-29T23:02:45.738800+0000","last_became_peered":"2023-03-29T23:02:45.738800+0000","last_unstale":"2023-03-29T23:02:55.831154+0000","last_undegraded":"2023-03-29T23:02:55.831154+0000","last_fullsized":"2023-03-29T23:02:55.831154+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:57:37.844148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.926232+0000","last_change":"2023-03-29T23:02:55.834753+0000","last_active":"2023-03-29T23:02:56.926232+0000","last_peered":"2023-03-29T23:02:56.926232+0000","last_clean":"2023-03-29T23:02:56.926232+0000","last_became_active":"2023-03-29T23:02:55.834476+0000","last_became_peered":"2023-03-29T23:02:55.834476+0000","last_unstale":"2023-03-29T23:02:56.926232+0000","last_undegraded":"2023-03-29T23:02:56.926232+0000","last_fullsized":"2023-03-29T23:02:56.926232+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:38:50.056810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.821960+0000","last_change":"2023-03-29T23:02:45.738002+0000","last_active":"2023-03-29T23:02:55.821960+0000","last_peered":"2023-03-29T23:02:55.821960+0000","last_clean":"2023-03-29T23:02:55.821960+0000","last_became_active":"2023-03-29T23:02:45.737619+0000","last_became_peered":"2023-03-29T23:02:45.737619+0000","last_unstale":"2023-03-29T23:02:55.821960+0000","last_undegraded":"2023-03-29T23:02:55.821960+0000","last_fullsized":"2023-03-29T23:02:55.821960+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:57.624010+0000","last_change":"2023-03-29T23:02:55.829208+0000","last_active":"2023-03-29T23:02:57.624010+0000","last_peered":"2023-03-29T23:02:57.624010+0000","last_clean":"2023-03-29T23:02:57.624010+0000","last_became_active":"2023-03-29T23:02:55.828918+0000","last_became_peered":"2023-03-29T23:02:55.828918+0000","last_unstale":"2023-03-29T23:02:57.624010+0000","last_undegraded":"2023-03-29T23:02:57.624010+0000","last_fullsized":"2023-03-29T23:02:57.624010+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:38:50.056810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"19'30","reported_seq":40,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:02.407987+0000","last_change":"2023-03-29T23:02:55.831980+0000","last_active":"2023-03-29T23:03:02.407987+0000","last_peered":"2023-03-29T23:03:02.407987+0000","last_clean":"2023-03-29T23:03:02.407987+0000","last_became_active":"2023-03-29T23:02:55.831680+0000","last_became_peered":"2023-03-29T23:02:55.831680+0000","last_unstale":"2023-03-29T23:03:02.407987+0000","last_undegraded":"2023-03-29T23:03:02.407987+0000","last_fullsized":"2023-03-29T23:03:02.407987+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:36:30.569179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827602+0000","last_change":"2023-03-29T23:02:45.772383+0000","last_active":"2023-03-29T23:02:55.827602+0000","last_peered":"2023-03-29T23:02:55.827602+0000","last_clean":"2023-03-29T23:02:55.827602+0000","last_became_active":"2023-03-29T23:02:45.771921+0000","last_became_peered":"2023-03-29T23:02:45.771921+0000","last_unstale":"2023-03-29T23:02:55.827602+0000","last_undegraded":"2023-03-29T23:02:55.827602+0000","last_fullsized":"2023-03-29T23:02:55.827602+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"18'5","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.926612+0000","last_change":"2023-03-29T23:02:55.835088+0000","last_active":"2023-03-29T23:02:56.926612+0000","last_peered":"2023-03-29T23:02:56.926612+0000","last_clean":"2023-03-29T23:02:56.926612+0000","last_became_active":"2023-03-29T23:02:55.834783+0000","last_became_peered":"2023-03-29T23:02:55.834783+0000","last_unstale":"2023-03-29T23:02:56.926612+0000","last_undegraded":"2023-03-29T23:02:56.926612+0000","last_fullsized":"2023-03-29T23:02:56.926612+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:48:37.198336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.825363+0000","last_change":"2023-03-29T23:02:45.798164+0000","last_active":"2023-03-29T23:02:55.825363+0000","last_peered":"2023-03-29T23:02:55.825363+0000","last_clean":"2023-03-29T23:02:55.825363+0000","last_became_active":"2023-03-29T23:02:45.797575+0000","last_became_peered":"2023-03-29T23:02:45.797575+0000","last_unstale":"2023-03-29T23:02:55.825363+0000","last_undegraded":"2023-03-29T23:02:55.825363+0000","last_fullsized":"2023-03-29T23:02:55.825363+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"19'82","reported_seq":92,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:18.624950+0000","last_change":"2023-03-29T23:02:55.830433+0000","last_active":"2023-03-29T23:03:18.624950+0000","last_peered":"2023-03-29T23:03:18.624950+0000","last_clean":"2023-03-29T23:03:18.624950+0000","last_became_active":"2023-03-29T23:02:55.830123+0000","last_became_peered":"2023-03-29T23:02:55.830123+0000","last_unstale":"2023-03-29T23:03:18.624950+0000","last_undegraded":"2023-03-29T23:03:18.624950+0000","last_fullsized":"2023-03-29T23:03:18.624950+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:41:47.370406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827704+0000","last_change":"2023-03-29T23:02:45.772391+0000","last_active":"2023-03-29T23:02:55.827704+0000","last_peered":"2023-03-29T23:02:55.827704+0000","last_clean":"2023-03-29T23:02:55.827704+0000","last_became_active":"2023-03-29T23:02:45.772083+0000","last_became_peered":"2023-03-29T23:02:45.772083+0000","last_unstale":"2023-03-29T23:02:55.827704+0000","last_undegraded":"2023-03-29T23:02:55.827704+0000","last_fullsized":"2023-03-29T23:02:55.827704+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:07:10.271259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"19'126","reported_seq":136,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:17.592920+0000","last_change":"2023-03-29T23:02:55.833133+0000","last_active":"2023-03-29T23:03:17.592920+0000","last_peered":"2023-03-29T23:03:17.592920+0000","last_clean":"2023-03-29T23:03:17.592920+0000","last_became_active":"2023-03-29T23:02:55.832902+0000","last_became_peered":"2023-03-29T23:02:55.832902+0000","last_unstale":"2023-03-29T23:03:17.592920+0000","last_undegraded":"2023-03-29T23:03:17.592920+0000","last_fullsized":"2023-03-29T23:03:17.592920+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:16:02.844424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.825445+0000","last_change":"2023-03-29T23:02:45.772640+0000","last_active":"2023-03-29T23:02:55.825445+0000","last_peered":"2023-03-29T23:02:55.825445+0000","last_clean":"2023-03-29T23:02:55.825445+0000","last_became_active":"2023-03-29T23:02:45.772192+0000","last_became_peered":"2023-03-29T23:02:45.772192+0000","last_unstale":"2023-03-29T23:02:55.825445+0000","last_undegraded":"2023-03-29T23:02:55.825445+0000","last_fullsized":"2023-03-29T23:02:55.825445+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:08:58.500583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":397,"num_write_kb":6541,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":397,"ondisk_log_size":397,"up":24,"acting":24,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640266,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6600,"kb_used_data":1472,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104851000,"statfs":{"total":107374182400,"available":107367424000,"internally_reserved":0,"allocated":1507328,"data_stored":1362193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94699999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96099999999999997}]}]},{"osd":4,"up_from":11,"seq":47244640266,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8016,"kb_used_data":2760,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849584,"statfs":{"total":107374182400,"available":107365974016,"internally_reserved":0,"allocated":2826240,"data_stored":2672193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81499999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.87}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71899999999999997}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8728,"kb_used_data":3408,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848872,"statfs":{"total":107374182400,"available":107365244928,"internally_reserved":0,"allocated":3489792,"data_stored":3332193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65900000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80500000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0149999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96899999999999997}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6908,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850692,"statfs":{"total":107374182400,"available":107367108608,"internally_reserved":0,"allocated":1822720,"data_stored":1672193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68999999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77900000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94699999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72799999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96499999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8648,"kb_used_data":3008,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848952,"statfs":{"total":107374182400,"available":107365326848,"internally_reserved":0,"allocated":3080192,"data_stored":2922561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69999999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8340,"kb_used_data":2700,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849260,"statfs":{"total":107374182400,"available":107365642240,"internally_reserved":0,"allocated":2764800,"data_stored":2612561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:24.771 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:46:24.771 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. , 2023-03-29T23:46:44.772 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_ceph_ceph-c_e369dd579b62fff69e4b88c7d4fb7419fe60653c/qa/tasks/ceph.py", line 1946, in task , osd_scrub_pgs(ctx, config) Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f4db4293420] , 2: gsignal() Ignoring malparsed backtrace: 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:35:43.121 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:35:43.122 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:35:43.122 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:36:30.130 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:36:30.130 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:36:30.131 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:38:38.470 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:38:38.470 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:38:38.471 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:39:59.778 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:39:59.778 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:39:59.779 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:40:46.811 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:40:46.812 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:40:46.813 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:42:08.118 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:08.118 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:08.119 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:42:55.129 INFO:tasks.ceph:pgid 3.6 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:55.129 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:55.130 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. Ignoring malparsed backtrace: 2023-03-29T23:44:16.449 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:44:16.449 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:44:16.450 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:45:23.759 INFO:teuthology.orchestra.run.smithi072.stdout: , dumped all , 2023-03-29T23:45:23.773 INFO:teuthology.orchestra.run.smithi072.stdout:{"pg_ready":true,"pg_map":{"version":1294,"stamp":"2023-03-29T23:45:22.955407+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":534,"num_write_kb":8435,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":494,"ondisk_log_size":494,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47240,"kb_used_data":15128,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098360,"statfs":{"total":644245094400,"available":644196720640,"internally_reserved":0,"allocated":15491072,"data_stored":14573894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003770"},"pg_stats":[{"pgid":"3.b","version":"19'26","reported_seq":36,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:06.497117+0000","last_change":"2023-03-29T23:02:55.829125+0000","last_active":"2023-03-29T23:03:06.497117+0000","last_peered":"2023-03-29T23:03:06.497117+0000","last_clean":"2023-03-29T23:03:06.497117+0000","last_became_active":"2023-03-29T23:02:55.828791+0000","last_became_peered":"2023-03-29T23:02:55.828791+0000","last_unstale":"2023-03-29T23:03:06.497117+0000","last_undegraded":"2023-03-29T23:03:06.497117+0000","last_fullsized":"2023-03-29T23:03:06.497117+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:14:34.429204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.828630+0000","last_change":"2023-03-29T23:02:55.833364+0000","last_active":"2023-03-29T23:02:56.828630+0000","last_peered":"2023-03-29T23:02:56.828630+0000","last_clean":"2023-03-29T23:02:56.828630+0000","last_became_active":"2023-03-29T23:02:55.833083+0000","last_became_peered":"2023-03-29T23:02:55.833083+0000","last_unstale":"2023-03-29T23:02:56.828630+0000","last_undegraded":"2023-03-29T23:02:56.828630+0000","last_fullsized":"2023-03-29T23:02:56.828630+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T09:30:35.161625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"19'64","reported_seq":74,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:11.546610+0000","last_change":"2023-03-29T23:02:55.834770+0000","last_active":"2023-03-29T23:03:11.546610+0000","last_peered":"2023-03-29T23:03:11.546610+0000","last_clean":"2023-03-29T23:03:11.546610+0000","last_became_active":"2023-03-29T23:02:55.834517+0000","last_became_peered":"2023-03-29T23:02:55.834517+0000","last_unstale":"2023-03-29T23:03:11.546610+0000","last_undegraded":"2023-03-29T23:03:11.546610+0000","last_fullsized":"2023-03-29T23:03:11.546610+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:14:34.429204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:57.623892+0000","last_change":"2023-03-29T23:02:55.832500+0000","last_active":"2023-03-29T23:02:57.623892+0000","last_peered":"2023-03-29T23:02:57.623892+0000","last_clean":"2023-03-29T23:02:57.623892+0000","last_became_active":"2023-03-29T23:02:55.832252+0000","last_became_peered":"2023-03-29T23:02:55.832252+0000","last_unstale":"2023-03-29T23:02:57.623892+0000","last_undegraded":"2023-03-29T23:02:57.623892+0000","last_fullsized":"2023-03-29T23:02:57.623892+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T09:30:35.161625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"14'97","reported_seq":141,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827817+0000","last_change":"2023-03-29T23:02:45.772706+0000","last_active":"2023-03-29T23:02:55.827817+0000","last_peered":"2023-03-29T23:02:55.827817+0000","last_clean":"2023-03-29T23:02:55.827817+0000","last_became_active":"2023-03-29T23:02:45.772124+0000","last_became_peered":"2023-03-29T23:02:45.772124+0000","last_unstale":"2023-03-29T23:02:55.827817+0000","last_undegraded":"2023-03-29T23:02:55.827817+0000","last_fullsized":"2023-03-29T23:02:55.827817+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:08:58.500583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.822203+0000","last_change":"2023-03-29T23:02:45.798160+0000","last_active":"2023-03-29T23:02:55.822203+0000","last_peered":"2023-03-29T23:02:55.822203+0000","last_clean":"2023-03-29T23:02:55.822203+0000","last_became_active":"2023-03-29T23:02:45.797639+0000","last_became_peered":"2023-03-29T23:02:45.797639+0000","last_unstale":"2023-03-29T23:02:55.822203+0000","last_undegraded":"2023-03-29T23:02:55.822203+0000","last_fullsized":"2023-03-29T23:02:55.822203+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T11:02:14.658224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"19'30","reported_seq":40,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:05.052479+0000","last_change":"2023-03-29T23:02:55.831017+0000","last_active":"2023-03-29T23:03:05.052479+0000","last_peered":"2023-03-29T23:03:05.052479+0000","last_clean":"2023-03-29T23:03:05.052479+0000","last_became_active":"2023-03-29T23:02:55.830688+0000","last_became_peered":"2023-03-29T23:02:55.830688+0000","last_unstale":"2023-03-29T23:03:05.052479+0000","last_undegraded":"2023-03-29T23:03:05.052479+0000","last_fullsized":"2023-03-29T23:03:05.052479+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:15:50.681015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.822940+0000","last_change":"2023-03-29T23:02:45.738317+0000","last_active":"2023-03-29T23:02:55.822940+0000","last_peered":"2023-03-29T23:02:55.822940+0000","last_clean":"2023-03-29T23:02:55.822940+0000","last_became_active":"2023-03-29T23:02:45.737938+0000","last_became_peered":"2023-03-29T23:02:45.737938+0000","last_unstale":"2023-03-29T23:02:55.822940+0000","last_undegraded":"2023-03-29T23:02:55.822940+0000","last_fullsized":"2023-03-29T23:02:55.822940+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:07:10.271259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"19'34","reported_seq":44,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:13.432024+0000","last_change":"2023-03-29T23:02:55.830034+0000","last_active":"2023-03-29T23:03:13.432024+0000","last_peered":"2023-03-29T23:03:13.432024+0000","last_clean":"2023-03-29T23:03:13.432024+0000","last_became_active":"2023-03-29T23:02:55.829749+0000","last_became_peered":"2023-03-29T23:02:55.829749+0000","last_unstale":"2023-03-29T23:03:13.432024+0000","last_undegraded":"2023-03-29T23:03:13.432024+0000","last_fullsized":"2023-03-29T23:03:13.432024+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:48:37.198336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.831154+0000","last_change":"2023-03-29T23:02:45.739333+0000","last_active":"2023-03-29T23:02:55.831154+0000","last_peered":"2023-03-29T23:02:55.831154+0000","last_clean":"2023-03-29T23:02:55.831154+0000","last_became_active":"2023-03-29T23:02:45.738800+0000","last_became_peered":"2023-03-29T23:02:45.738800+0000","last_unstale":"2023-03-29T23:02:55.831154+0000","last_undegraded":"2023-03-29T23:02:55.831154+0000","last_fullsized":"2023-03-29T23:02:55.831154+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:57:37.844148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.926232+0000","last_change":"2023-03-29T23:02:55.834753+0000","last_active":"2023-03-29T23:02:56.926232+0000","last_peered":"2023-03-29T23:02:56.926232+0000","last_clean":"2023-03-29T23:02:56.926232+0000","last_became_active":"2023-03-29T23:02:55.834476+0000","last_became_peered":"2023-03-29T23:02:55.834476+0000","last_unstale":"2023-03-29T23:02:56.926232+0000","last_undegraded":"2023-03-29T23:02:56.926232+0000","last_fullsized":"2023-03-29T23:02:56.926232+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:38:50.056810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.821960+0000","last_change":"2023-03-29T23:02:45.738002+0000","last_active":"2023-03-29T23:02:55.821960+0000","last_peered":"2023-03-29T23:02:55.821960+0000","last_clean":"2023-03-29T23:02:55.821960+0000","last_became_active":"2023-03-29T23:02:45.737619+0000","last_became_peered":"2023-03-29T23:02:45.737619+0000","last_unstale":"2023-03-29T23:02:55.821960+0000","last_undegraded":"2023-03-29T23:02:55.821960+0000","last_fullsized":"2023-03-29T23:02:55.821960+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:57.624010+0000","last_change":"2023-03-29T23:02:55.829208+0000","last_active":"2023-03-29T23:02:57.624010+0000","last_peered":"2023-03-29T23:02:57.624010+0000","last_clean":"2023-03-29T23:02:57.624010+0000","last_became_active":"2023-03-29T23:02:55.828918+0000","last_became_peered":"2023-03-29T23:02:55.828918+0000","last_unstale":"2023-03-29T23:02:57.624010+0000","last_undegraded":"2023-03-29T23:02:57.624010+0000","last_fullsized":"2023-03-29T23:02:57.624010+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:38:50.056810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"19'30","reported_seq":40,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:02.407987+0000","last_change":"2023-03-29T23:02:55.831980+0000","last_active":"2023-03-29T23:03:02.407987+0000","last_peered":"2023-03-29T23:03:02.407987+0000","last_clean":"2023-03-29T23:03:02.407987+0000","last_became_active":"2023-03-29T23:02:55.831680+0000","last_became_peered":"2023-03-29T23:02:55.831680+0000","last_unstale":"2023-03-29T23:03:02.407987+0000","last_undegraded":"2023-03-29T23:03:02.407987+0000","last_fullsized":"2023-03-29T23:03:02.407987+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:36:30.569179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827602+0000","last_change":"2023-03-29T23:02:45.772383+0000","last_active":"2023-03-29T23:02:55.827602+0000","last_peered":"2023-03-29T23:02:55.827602+0000","last_clean":"2023-03-29T23:02:55.827602+0000","last_became_active":"2023-03-29T23:02:45.771921+0000","last_became_peered":"2023-03-29T23:02:45.771921+0000","last_unstale":"2023-03-29T23:02:55.827602+0000","last_undegraded":"2023-03-29T23:02:55.827602+0000","last_fullsized":"2023-03-29T23:02:55.827602+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"18'5","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.926612+0000","last_change":"2023-03-29T23:02:55.835088+0000","last_active":"2023-03-29T23:02:56.926612+0000","last_peered":"2023-03-29T23:02:56.926612+0000","last_clean":"2023-03-29T23:02:56.926612+0000","last_became_active":"2023-03-29T23:02:55.834783+0000","last_became_peered":"2023-03-29T23:02:55.834783+0000","last_unstale":"2023-03-29T23:02:56.926612+0000","last_undegraded":"2023-03-29T23:02:56.926612+0000","last_fullsized":"2023-03-29T23:02:56.926612+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:48:37.198336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.825363+0000","last_change":"2023-03-29T23:02:45.798164+0000","last_active":"2023-03-29T23:02:55.825363+0000","last_peered":"2023-03-29T23:02:55.825363+0000","last_clean":"2023-03-29T23:02:55.825363+0000","last_became_active":"2023-03-29T23:02:45.797575+0000","last_became_peered":"2023-03-29T23:02:45.797575+0000","last_unstale":"2023-03-29T23:02:55.825363+0000","last_undegraded":"2023-03-29T23:02:55.825363+0000","last_fullsized":"2023-03-29T23:02:55.825363+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"19'82","reported_seq":92,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:18.624950+0000","last_change":"2023-03-29T23:02:55.830433+0000","last_active":"2023-03-29T23:03:18.624950+0000","last_peered":"2023-03-29T23:03:18.624950+0000","last_clean":"2023-03-29T23:03:18.624950+0000","last_became_active":"2023-03-29T23:02:55.830123+0000","last_became_peered":"2023-03-29T23:02:55.830123+0000","last_unstale":"2023-03-29T23:03:18.624950+0000","last_undegraded":"2023-03-29T23:03:18.624950+0000","last_fullsized":"2023-03-29T23:03:18.624950+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:41:47.370406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827704+0000","last_change":"2023-03-29T23:02:45.772391+0000","last_active":"2023-03-29T23:02:55.827704+0000","last_peered":"2023-03-29T23:02:55.827704+0000","last_clean":"2023-03-29T23:02:55.827704+0000","last_became_active":"2023-03-29T23:02:45.772083+0000","last_became_peered":"2023-03-29T23:02:45.772083+0000","last_unstale":"2023-03-29T23:02:55.827704+0000","last_undegraded":"2023-03-29T23:02:55.827704+0000","last_fullsized":"2023-03-29T23:02:55.827704+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:07:10.271259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"19'126","reported_seq":136,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:17.592920+0000","last_change":"2023-03-29T23:02:55.833133+0000","last_active":"2023-03-29T23:03:17.592920+0000","last_peered":"2023-03-29T23:03:17.592920+0000","last_clean":"2023-03-29T23:03:17.592920+0000","last_became_active":"2023-03-29T23:02:55.832902+0000","last_became_peered":"2023-03-29T23:02:55.832902+0000","last_unstale":"2023-03-29T23:03:17.592920+0000","last_undegraded":"2023-03-29T23:03:17.592920+0000","last_fullsized":"2023-03-29T23:03:17.592920+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:16:02.844424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.825445+0000","last_change":"2023-03-29T23:02:45.772640+0000","last_active":"2023-03-29T23:02:55.825445+0000","last_peered":"2023-03-29T23:02:55.825445+0000","last_clean":"2023-03-29T23:02:55.825445+0000","last_became_active":"2023-03-29T23:02:45.772192+0000","last_became_peered":"2023-03-29T23:02:45.772192+0000","last_unstale":"2023-03-29T23:02:55.825445+0000","last_undegraded":"2023-03-29T23:02:55.825445+0000","last_fullsized":"2023-03-29T23:02:55.825445+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:08:58.500583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":397,"num_write_kb":6541,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":397,"ondisk_log_size":397,"up":24,"acting":24,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640266,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6600,"kb_used_data":1472,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104851000,"statfs":{"total":107374182400,"available":107367424000,"internally_reserved":0,"allocated":1507328,"data_stored":1362193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94699999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96099999999999997}]}]},{"osd":4,"up_from":11,"seq":47244640266,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8016,"kb_used_data":2760,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849584,"statfs":{"total":107374182400,"available":107365974016,"internally_reserved":0,"allocated":2826240,"data_stored":2672193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81499999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.87}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71899999999999997}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8728,"kb_used_data":3408,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848872,"statfs":{"total":107374182400,"available":107365244928,"internally_reserved":0,"allocated":3489792,"data_stored":3332193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65900000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80500000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0149999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96899999999999997}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6908,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850692,"statfs":{"total":107374182400,"available":107367108608,"internally_reserved":0,"allocated":1822720,"data_stored":1672193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68999999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77900000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94699999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72799999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96499999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8648,"kb_used_data":3008,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848952,"statfs":{"total":107374182400,"available":107365326848,"internally_reserved":0,"allocated":3080192,"data_stored":2922561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69999999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8340,"kb_used_data":2700,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849260,"statfs":{"total":107374182400,"available":107365642240,"internally_reserved":0,"allocated":2764800,"data_stored":2612561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:24.771 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:46:24.771 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. , 2023-03-29T23:46:44.772 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_ceph_ceph-c_e369dd579b62fff69e4b88c7d4fb7419fe60653c/qa/tasks/ceph.py", line 1946, in task , osd_scrub_pgs(ctx, config) Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7f4db4293420] , 2: gsignal() Ignoring malparsed backtrace: 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:34:21.803 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:35:43.121 INFO:tasks.ceph:pgid 3.b last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:35:43.122 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:35:43.122 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:36:30.130 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:36:30.130 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:36:30.131 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:37:51.435 INFO:tasks.ceph:pgid 3.2 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:38:38.470 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:38:38.470 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:38:38.471 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:39:59.778 INFO:tasks.ceph:pgid 3.1 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:39:59.778 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:39:59.779 INFO:tasks.ceph:pgid 3.0 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:40:46.811 INFO:tasks.ceph:pgid 3.4 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:40:46.812 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:40:46.813 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:42:08.118 INFO:tasks.ceph:pgid 3.3 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:08.118 INFO:tasks.ceph:pgid 3.5 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:08.119 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:42:55.129 INFO:tasks.ceph:pgid 3.6 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:55.129 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:42:55.130 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. Ignoring malparsed backtrace: 2023-03-29T23:44:16.449 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:44:16.449 INFO:tasks.ceph:pgid 3.7 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:44:16.450 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: 2023-03-29T23:45:23.759 INFO:teuthology.orchestra.run.smithi072.stdout: , dumped all , 2023-03-29T23:45:23.773 INFO:teuthology.orchestra.run.smithi072.stdout:{"pg_ready":true,"pg_map":{"version":1294,"stamp":"2023-03-29T23:45:22.955407+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7100368,"num_objects":22,"num_object_clones":0,"num_object_copies":44,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":534,"num_write_kb":8435,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":494,"ondisk_log_size":494,"up":42,"acting":42,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":42,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":629145600,"kb_used":47240,"kb_used_data":15128,"kb_used_omap":0,"kb_used_meta":32064,"kb_avail":629098360,"statfs":{"total":644245094400,"available":644196720640,"internally_reserved":0,"allocated":15491072,"data_stored":14573894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":32833536},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003770"},"pg_stats":[{"pgid":"3.b","version":"19'26","reported_seq":36,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:06.497117+0000","last_change":"2023-03-29T23:02:55.829125+0000","last_active":"2023-03-29T23:03:06.497117+0000","last_peered":"2023-03-29T23:03:06.497117+0000","last_clean":"2023-03-29T23:03:06.497117+0000","last_became_active":"2023-03-29T23:02:55.828791+0000","last_became_peered":"2023-03-29T23:02:55.828791+0000","last_unstale":"2023-03-29T23:03:06.497117+0000","last_undegraded":"2023-03-29T23:03:06.497117+0000","last_fullsized":"2023-03-29T23:03:06.497117+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":26,"log_dups_size":0,"ondisk_log_size":26,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:14:34.429204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26,"num_write_kb":335,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.828630+0000","last_change":"2023-03-29T23:02:55.833364+0000","last_active":"2023-03-29T23:02:56.828630+0000","last_peered":"2023-03-29T23:02:56.828630+0000","last_clean":"2023-03-29T23:02:56.828630+0000","last_became_active":"2023-03-29T23:02:55.833083+0000","last_became_peered":"2023-03-29T23:02:55.833083+0000","last_unstale":"2023-03-29T23:02:56.828630+0000","last_undegraded":"2023-03-29T23:02:56.828630+0000","last_fullsized":"2023-03-29T23:02:56.828630+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T09:30:35.161625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"19'64","reported_seq":74,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:11.546610+0000","last_change":"2023-03-29T23:02:55.834770+0000","last_active":"2023-03-29T23:03:11.546610+0000","last_peered":"2023-03-29T23:03:11.546610+0000","last_clean":"2023-03-29T23:03:11.546610+0000","last_became_active":"2023-03-29T23:02:55.834517+0000","last_became_peered":"2023-03-29T23:02:55.834517+0000","last_unstale":"2023-03-29T23:03:11.546610+0000","last_undegraded":"2023-03-29T23:03:11.546610+0000","last_fullsized":"2023-03-29T23:03:11.546610+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":64,"log_dups_size":0,"ondisk_log_size":64,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:14:34.429204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":970000,"num_objects":3,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":978,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:57.623892+0000","last_change":"2023-03-29T23:02:55.832500+0000","last_active":"2023-03-29T23:02:57.623892+0000","last_peered":"2023-03-29T23:02:57.623892+0000","last_clean":"2023-03-29T23:02:57.623892+0000","last_became_active":"2023-03-29T23:02:55.832252+0000","last_became_peered":"2023-03-29T23:02:55.832252+0000","last_unstale":"2023-03-29T23:02:57.623892+0000","last_undegraded":"2023-03-29T23:02:57.623892+0000","last_fullsized":"2023-03-29T23:02:57.623892+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T09:30:35.161625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.0","version":"14'97","reported_seq":141,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827817+0000","last_change":"2023-03-29T23:02:45.772706+0000","last_active":"2023-03-29T23:02:55.827817+0000","last_peered":"2023-03-29T23:02:55.827817+0000","last_clean":"2023-03-29T23:02:55.827817+0000","last_became_active":"2023-03-29T23:02:45.772124+0000","last_became_peered":"2023-03-29T23:02:45.772124+0000","last_unstale":"2023-03-29T23:02:55.827817+0000","last_undegraded":"2023-03-29T23:02:55.827817+0000","last_fullsized":"2023-03-29T23:02:55.827817+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:08:58.500583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.822203+0000","last_change":"2023-03-29T23:02:45.798160+0000","last_active":"2023-03-29T23:02:55.822203+0000","last_peered":"2023-03-29T23:02:55.822203+0000","last_clean":"2023-03-29T23:02:55.822203+0000","last_became_active":"2023-03-29T23:02:45.797639+0000","last_became_peered":"2023-03-29T23:02:45.797639+0000","last_unstale":"2023-03-29T23:02:55.822203+0000","last_undegraded":"2023-03-29T23:02:55.822203+0000","last_fullsized":"2023-03-29T23:02:55.822203+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T11:02:14.658224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"19'30","reported_seq":40,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:05.052479+0000","last_change":"2023-03-29T23:02:55.831017+0000","last_active":"2023-03-29T23:03:05.052479+0000","last_peered":"2023-03-29T23:03:05.052479+0000","last_clean":"2023-03-29T23:03:05.052479+0000","last_became_active":"2023-03-29T23:02:55.830688+0000","last_became_peered":"2023-03-29T23:02:55.830688+0000","last_unstale":"2023-03-29T23:03:05.052479+0000","last_undegraded":"2023-03-29T23:03:05.052479+0000","last_fullsized":"2023-03-29T23:03:05.052479+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:15:50.681015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.822940+0000","last_change":"2023-03-29T23:02:45.738317+0000","last_active":"2023-03-29T23:02:55.822940+0000","last_peered":"2023-03-29T23:02:55.822940+0000","last_clean":"2023-03-29T23:02:55.822940+0000","last_became_active":"2023-03-29T23:02:45.737938+0000","last_became_peered":"2023-03-29T23:02:45.737938+0000","last_unstale":"2023-03-29T23:02:55.822940+0000","last_undegraded":"2023-03-29T23:02:55.822940+0000","last_fullsized":"2023-03-29T23:02:55.822940+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:07:10.271259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"19'34","reported_seq":44,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:13.432024+0000","last_change":"2023-03-29T23:02:55.830034+0000","last_active":"2023-03-29T23:03:13.432024+0000","last_peered":"2023-03-29T23:03:13.432024+0000","last_clean":"2023-03-29T23:03:13.432024+0000","last_became_active":"2023-03-29T23:02:55.829749+0000","last_became_peered":"2023-03-29T23:02:55.829749+0000","last_unstale":"2023-03-29T23:03:13.432024+0000","last_undegraded":"2023-03-29T23:03:13.432024+0000","last_fullsized":"2023-03-29T23:03:13.432024+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":34,"log_dups_size":0,"ondisk_log_size":34,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:48:37.198336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330000,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":34,"num_write_kb":339,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.831154+0000","last_change":"2023-03-29T23:02:45.739333+0000","last_active":"2023-03-29T23:02:55.831154+0000","last_peered":"2023-03-29T23:02:55.831154+0000","last_clean":"2023-03-29T23:02:55.831154+0000","last_became_active":"2023-03-29T23:02:45.738800+0000","last_became_peered":"2023-03-29T23:02:45.738800+0000","last_unstale":"2023-03-29T23:02:55.831154+0000","last_undegraded":"2023-03-29T23:02:55.831154+0000","last_fullsized":"2023-03-29T23:02:55.831154+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:57:37.844148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.926232+0000","last_change":"2023-03-29T23:02:55.834753+0000","last_active":"2023-03-29T23:02:56.926232+0000","last_peered":"2023-03-29T23:02:56.926232+0000","last_clean":"2023-03-29T23:02:56.926232+0000","last_became_active":"2023-03-29T23:02:55.834476+0000","last_became_peered":"2023-03-29T23:02:55.834476+0000","last_unstale":"2023-03-29T23:02:56.926232+0000","last_undegraded":"2023-03-29T23:02:56.926232+0000","last_fullsized":"2023-03-29T23:02:56.926232+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:38:50.056810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.821960+0000","last_change":"2023-03-29T23:02:45.738002+0000","last_active":"2023-03-29T23:02:55.821960+0000","last_peered":"2023-03-29T23:02:55.821960+0000","last_clean":"2023-03-29T23:02:55.821960+0000","last_became_active":"2023-03-29T23:02:45.737619+0000","last_became_peered":"2023-03-29T23:02:45.737619+0000","last_unstale":"2023-03-29T23:02:55.821960+0000","last_undegraded":"2023-03-29T23:02:55.821960+0000","last_fullsized":"2023-03-29T23:02:55.821960+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":10,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:57.624010+0000","last_change":"2023-03-29T23:02:55.829208+0000","last_active":"2023-03-29T23:02:57.624010+0000","last_peered":"2023-03-29T23:02:57.624010+0000","last_clean":"2023-03-29T23:02:57.624010+0000","last_became_active":"2023-03-29T23:02:55.828918+0000","last_became_peered":"2023-03-29T23:02:55.828918+0000","last_unstale":"2023-03-29T23:02:57.624010+0000","last_undegraded":"2023-03-29T23:02:57.624010+0000","last_fullsized":"2023-03-29T23:02:57.624010+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T02:38:50.056810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.5","version":"19'30","reported_seq":40,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:02.407987+0000","last_change":"2023-03-29T23:02:55.831980+0000","last_active":"2023-03-29T23:03:02.407987+0000","last_peered":"2023-03-29T23:03:02.407987+0000","last_clean":"2023-03-29T23:03:02.407987+0000","last_became_active":"2023-03-29T23:02:55.831680+0000","last_became_peered":"2023-03-29T23:02:55.831680+0000","last_unstale":"2023-03-29T23:03:02.407987+0000","last_undegraded":"2023-03-29T23:03:02.407987+0000","last_fullsized":"2023-03-29T23:03:02.407987+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":30,"log_dups_size":0,"ondisk_log_size":30,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T05:36:30.569179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":650000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30,"num_write_kb":649,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":14,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827602+0000","last_change":"2023-03-29T23:02:45.772383+0000","last_active":"2023-03-29T23:02:55.827602+0000","last_peered":"2023-03-29T23:02:55.827602+0000","last_clean":"2023-03-29T23:02:55.827602+0000","last_became_active":"2023-03-29T23:02:45.771921+0000","last_became_peered":"2023-03-29T23:02:45.771921+0000","last_unstale":"2023-03-29T23:02:55.827602+0000","last_undegraded":"2023-03-29T23:02:55.827602+0000","last_fullsized":"2023-03-29T23:02:55.827602+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"18'5","reported_seq":15,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:02:56.926612+0000","last_change":"2023-03-29T23:02:55.835088+0000","last_active":"2023-03-29T23:02:56.926612+0000","last_peered":"2023-03-29T23:02:56.926612+0000","last_clean":"2023-03-29T23:02:56.926612+0000","last_became_active":"2023-03-29T23:02:55.834783+0000","last_became_peered":"2023-03-29T23:02:55.834783+0000","last_unstale":"2023-03-29T23:02:56.926612+0000","last_undegraded":"2023-03-29T23:02:56.926612+0000","last_fullsized":"2023-03-29T23:02:56.926612+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:48:37.198336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":640000,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.825363+0000","last_change":"2023-03-29T23:02:45.798164+0000","last_active":"2023-03-29T23:02:55.825363+0000","last_peered":"2023-03-29T23:02:55.825363+0000","last_clean":"2023-03-29T23:02:55.825363+0000","last_became_active":"2023-03-29T23:02:45.797575+0000","last_became_peered":"2023-03-29T23:02:45.797575+0000","last_unstale":"2023-03-29T23:02:55.825363+0000","last_undegraded":"2023-03-29T23:02:55.825363+0000","last_fullsized":"2023-03-29T23:02:55.825363+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:34:57.344679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.7","version":"19'82","reported_seq":92,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:18.624950+0000","last_change":"2023-03-29T23:02:55.830433+0000","last_active":"2023-03-29T23:03:18.624950+0000","last_peered":"2023-03-29T23:03:18.624950+0000","last_clean":"2023-03-29T23:03:18.624950+0000","last_became_active":"2023-03-29T23:02:55.830123+0000","last_became_peered":"2023-03-29T23:02:55.830123+0000","last_unstale":"2023-03-29T23:03:18.624950+0000","last_undegraded":"2023-03-29T23:03:18.624950+0000","last_fullsized":"2023-03-29T23:03:18.624950+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":82,"log_dups_size":0,"ondisk_log_size":82,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:41:47.370406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310000,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82,"num_write_kb":1311,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":14,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.827704+0000","last_change":"2023-03-29T23:02:45.772391+0000","last_active":"2023-03-29T23:02:55.827704+0000","last_peered":"2023-03-29T23:02:55.827704+0000","last_clean":"2023-03-29T23:02:55.827704+0000","last_became_active":"2023-03-29T23:02:45.772083+0000","last_became_peered":"2023-03-29T23:02:45.772083+0000","last_unstale":"2023-03-29T23:02:55.827704+0000","last_undegraded":"2023-03-29T23:02:55.827704+0000","last_fullsized":"2023-03-29T23:02:55.827704+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:07:10.271259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.6","version":"19'126","reported_seq":136,"reported_epoch":19,"state":"active+clean","last_fresh":"2023-03-29T23:03:17.592920+0000","last_change":"2023-03-29T23:02:55.833133+0000","last_active":"2023-03-29T23:03:17.592920+0000","last_peered":"2023-03-29T23:03:17.592920+0000","last_clean":"2023-03-29T23:03:17.592920+0000","last_became_active":"2023-03-29T23:02:55.832902+0000","last_became_peered":"2023-03-29T23:02:55.832902+0000","last_unstale":"2023-03-29T23:03:17.592920+0000","last_undegraded":"2023-03-29T23:03:17.592920+0000","last_fullsized":"2023-03-29T23:03:17.592920+0000","mapping_epoch":17,"log_start":"0'0","ondisk_log_start":"0'0","created":17,"last_epoch_clean":18,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:54.802310+0000","last_clean_scrub_stamp":"2023-03-29T23:02:54.802310+0000","objects_scrubbed":0,"log_size":126,"log_dups_size":0,"ondisk_log_size":126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:16:02.844424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1630000,"num_objects":5,"num_object_clones":0,"num_object_copies":10,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":126,"num_write_kb":1653,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":5,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":18,"state":"active+clean","last_fresh":"2023-03-29T23:02:55.825445+0000","last_change":"2023-03-29T23:02:45.772640+0000","last_active":"2023-03-29T23:02:55.825445+0000","last_peered":"2023-03-29T23:02:55.825445+0000","last_clean":"2023-03-29T23:02:55.825445+0000","last_became_active":"2023-03-29T23:02:45.772192+0000","last_became_peered":"2023-03-29T23:02:45.772192+0000","last_unstale":"2023-03-29T23:02:55.825445+0000","last_undegraded":"2023-03-29T23:02:55.825445+0000","last_fullsized":"2023-03-29T23:02:55.825445+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:02:44.709278+0000","last_clean_scrub_stamp":"2023-03-29T23:02:44.709278+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:08:58.500583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":12,"stat_sum":{"num_bytes":6510000,"num_objects":20,"num_object_clones":0,"num_object_copies":40,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":397,"num_write_kb":6541,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":18,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13123584,"data_stored":13020000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":397,"ondisk_log_size":397,"up":24,"acting":24,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":6},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":106,"num_read_kb":91,"num_write":137,"num_write_kb":1894,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97,"ondisk_log_size":97,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":5,"up_from":11,"seq":47244640266,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6600,"kb_used_data":1472,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104851000,"statfs":{"total":107374182400,"available":107367424000,"internally_reserved":0,"allocated":1507328,"data_stored":1362193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67400000000000004}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94699999999999995}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96099999999999997}]}]},{"osd":4,"up_from":11,"seq":47244640266,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8016,"kb_used_data":2760,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":104849584,"statfs":{"total":107374182400,"available":107365974016,"internally_reserved":0,"allocated":2826240,"data_stored":2672193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.81499999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.501}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.87}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71899999999999997}]}]},{"osd":3,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8728,"kb_used_data":3408,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":104848872,"statfs":{"total":107374182400,"available":107365244928,"internally_reserved":0,"allocated":3489792,"data_stored":3332193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65900000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80500000000000005}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.0149999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96899999999999997}]}]},{"osd":2,"up_from":11,"seq":47244640266,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":6908,"kb_used_data":1780,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":104850692,"statfs":{"total":107374182400,"available":107367108608,"internally_reserved":0,"allocated":1822720,"data_stored":1672193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68999999999999995}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77900000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.94699999999999995}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72799999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.96499999999999997}]}]},{"osd":1,"up_from":11,"seq":47244640266,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8648,"kb_used_data":3008,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104848952,"statfs":{"total":107374182400,"available":107365326848,"internally_reserved":0,"allocated":3080192,"data_stored":2922561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[0,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.72099999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.69999999999999996}]}]},{"osd":0,"up_from":11,"seq":47244640265,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":104857600,"kb_used":8340,"kb_used_data":2700,"kb_used_omap":0,"kb_used_meta":5632,"kb_avail":104849260,"statfs":{"total":107374182400,"available":107365642240,"internally_reserved":0,"allocated":2764800,"data_stored":2612561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5767168},"hb_peers":[1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.437}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71499999999999997}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80000000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.85199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1974272,"data_stored":1960000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2289664,"data_stored":2270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1626112,"data_stored":1610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3293184,"data_stored":3270000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2629632,"data_stored":2610000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1310720,"data_stored":1300000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:24.771 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2023-03-29T23:02:44.709278+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=44, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:46:24.771 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. , 2023-03-29T23:46:44.772 DEBUG:teuthology.orchestra.run.smithi072:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.a last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.9 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) , 2023-03-29T23:47:32.092 INFO:tasks.ceph:pgid 3.8 last_scrub_stamp 2023-03-29T23:02:54.802310+0000 time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=2, tm_sec=54, tm_wday=2, tm_yday=88, tm_isdst=-1) <= time.struct_time(tm_year=2023, tm_mon=3, tm_mday=29, tm_hour=23, tm_min=22, tm_sec=20, tm_wday=2, tm_yday=88, tm_isdst=0) Ignoring malparsed backtrace: yield vars , File "/home/teuthworker/src/github.com_ceph_ceph-c_e369dd579b62fff69e4b88c7d4fb7419fe60653c/qa/tasks/ceph.py", line 1946, in task , osd_scrub_pgs(ctx, config) Misunderstood line: Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x135) [0x55a59601f2ad] , 2: ceph-osd(+0x631473) [0x55a59601f473] Ignoring malparsed backtrace: 2023-03-30T00:59:43.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:58:48.149595+0000", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op must wait for map", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.575214+0000", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7829999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 26.474046659999999, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062746649999999996, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:16.606341+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_started", , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:23.708494+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:23.712400+0000", , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.367927426000001, , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076374399999999997, Ignoring malparsed backtrace: 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010535 , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:541 3.1es0 3:7ef085b3:::smithi20399497-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:26.715419+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703128+0000", , 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2414e-05 , 2023-03-30T00:59:43.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703333+0000", , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012205999999999999 , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.774152+0000", , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00084599299999999996 , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, , 2023-03-30T00:59:43.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, Ignoring malparsed backtrace: 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0399999999999998e-06 , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.585362+0000", , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.614174+0000", , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7903e-05 , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.678976+0000", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.683130+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.690158+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpWrite(3.as5 71/54 ECSubWrite(tid=203, reqid=client.4555.0:735, at_version=71'38, trim_to=0'0, roll_forward_to=71'35))", , 2023-03-30T00:59:43.801 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.774378+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.807553+0000", , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012399939999999999 , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.812386+0000", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 Ignoring malparsed backtrace: 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 43 v 48 (ObjNum 129 snap 24 seq_num 129) dirty exists , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:300: left oid 43 (ObjNum 129 snap 24 seq_num 129) , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:302: finishing write tid 1 to smithi20399497-3 Ignoring malparsed backtrace: 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.581882+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.587173+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068317159999999998 , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:18.626041+0000", , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.059e-05 , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.685149+0000", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.758157+0000", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068448800000000002 Ignoring malparsed backtrace: 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.809042+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.440 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", Ignoring malparsed backtrace: 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:44.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00063433800000000002 , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.617 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768302+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.773+0000 7f7e20ae9700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:756d2f96:::smithi20399497-24:c repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:c63dffb4:::smithi20399497-40:6 repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{12=[12,10,d]} for object 3:cb5d7287:::smithi20399497-32:c Ignoring malparsed backtrace: 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.600924+0000", , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000208836 , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:44.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665069+0000", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665141+0000", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2509999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:44.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703115+0000", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0409000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.743640+0000", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076826619999999998 Ignoring malparsed backtrace: 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.755311+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.766273+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.4555", , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", Ignoring malparsed backtrace: 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", Ignoring malparsed backtrace: 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9876999999999993e-05 , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.792080+0000", , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8390000000000003e-06 , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:744 3.10s0 3:0aab8796:::smithi20399497-28:head [read 0~0,checksum] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e71)", Ignoring malparsed backtrace: 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.814167+0000", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2109999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:45.731+0000 7efdb31a1700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 11 (PID: 94971) UID: 0 , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:307: done (2 left) , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:308: rollback oid 45 current snap is 25 Ignoring malparsed backtrace: 2023-03-30T00:59:46.791+0000 7f2881c66700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:ef8379c0:::smithi20399497-18:5 repair needed (no obc) , 2023-03-30T00:59:46.791+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{11=[11,10,f,d]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:46.789+0000 7f579d0a4700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:5d617aee:::smithi20399497-20:3 repair needed (no obc) Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T00:59:47.581 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true , 2023-03-30T00:59:47.650+0000 7f0f9634b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 95055) UID: 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.5651e-05 , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0048024890000000001 , 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.888 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.900 INFO:teuthology.orchestra.run.smithi035.stdout: "tid": 249 , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.605832+0000", , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0661e-05 , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.926 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.618433+0000", , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.2633999999999995e-05 , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707573+0000", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707643+0000", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.7330000000000003e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "reached_pg", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680627+0000", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010247900000000001 Ignoring malparsed backtrace: 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.690506+0000", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.9204999999999997e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705513+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "op must wait for map", , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705691+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.710594+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.717126+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:48.075 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} , 2023-03-30T00:59:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 , 2023-03-30T00:59:48.076 INFO:tasks.rados.rados.0.smithi203.stdout:324: writing smithi20399497-19 from 2293760 to 3047424 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "complaint_time": 30, , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "num_blocked_ops": 0 , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010960500000000001 , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:527 3.14s0 3:2e5a366f:::smithi20399497-31:head [append 0~0] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:26.695461+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.698204+0000", , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.704831+0000", , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 9.7558000000000003e-05 , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:534 3.4s0 3:24eec11e:::smithi20399497-7:head [setxattr _header (56) in=7b] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", Ignoring malparsed backtrace: 2023-03-30T00:59:48.646 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.705518+0000", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.691686+0000", , 2023-03-30T00:59:48.671 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.01079631 Ignoring malparsed backtrace: 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680770+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.698 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "started", , 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "client_info": { , 2023-03-30T00:59:48.712 INFO:teuthology.orchestra.run.smithi035.stdout: "client": "client.4555", Ignoring malparsed backtrace: 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.2860000000000002e-06 , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0061165480000000003 , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: } , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:48.833+0000 7efd99558700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:14c33e9f:::smithi20399497-34:3 repair needed (no obc) , 2023-03-30T00:59:48.836 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 0 to 802816 tid 1 , 2023-03-30T00:59:48.838 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 802816 to 1228800 tid 2 Ignoring malparsed backtrace: 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ , 2023-03-30T00:59:49.307 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0722e-05 , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.604113+0000", , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00052637799999999998 , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0021209829999999999, , 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "type_data": { , 2023-03-30T00:59:49.363 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.610251+0000", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.7749999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "all_read", , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.608123+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:17.611077+0000", , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "age": 31.691323830999998, , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0017984959999999999, Ignoring malparsed backtrace: 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.789051+0000", , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.796145+0000", , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0040419669999999996 , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.491 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.798914+0000", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.9406e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 523, in _write_all , count = self._write(data) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1364, in _write Ignoring malparsed backtrace: 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "initiated", , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.560616+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.583527+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.2229999999999996e-06 , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.683455+0000", , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.7909999999999999e-06 , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.680363+0000", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.687466+0000", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.7453e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:50.135 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.765837+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.1ds1 77/54 ECSubWrite(tid=239, reqid=client.4555.0:880, at_version=77'66, trim_to=0'0, roll_forward_to=77'64))", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:48.854994+0000", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 1.1714130300000001, Ignoring malparsed backtrace: 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.8422000000000001e-05 , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.486+0000 7f6a8da31700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 14 (PID: 94707) UID: 0 , 2023-03-30T00:59:50.492 INFO:teuthology.orchestra.run.smithi035.stdout:osd.9: {} , 2023-03-30T00:59:50.513 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} Ignoring malparsed backtrace: 2023-03-30T00:59:51.270 INFO:teuthology.orchestra.run.smithi083.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:51.271 INFO:teuthology.orchestra.run.smithi083.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. , 2023-03-30T00:59:51.280 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops Ignoring malparsed backtrace: 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 1 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 2 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:348: finishing copy_from to smithi20399497-15 Ignoring malparsed backtrace: 2023-03-30T00:59:53.011+0000 7f57b74ee700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 13 (PID: 94715) UID: 0 , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. Ignoring malparsed backtrace: 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.599544+0000", , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0852000000000005e-05 , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.684493+0000", , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121767 , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.040131732000000003, , 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { , 2023-03-30T00:59:53.381 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2023-03-30T00:59:53.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703140+0000", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.756116+0000", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105009 Ignoring malparsed backtrace: 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:53.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779826+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779943+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: ] , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0249999999999998e-06 , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpReadReply(3.9s0 71/42 ECSubReadReply(tid=160, attrs_read=0))", , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.803584+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768289+0000", , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0489999999999999e-06 , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.521 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 756 , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:48.845453+0000", , 2023-03-30T00:59:53.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.843+0000 7fc5039f5700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 8 (PID: 95060) UID: 0 , 2023-03-30T00:59:53.847 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 13 1 , 2023-03-30T00:59:53.884+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0117e-05 , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.654122+0000", , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.688971+0000", , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0037431610000000001 , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.015 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.710655+0000", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.6818000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.743870+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.041 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.754628+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0947e-05 , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.813138+0000", , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.9743999999999999e-05 , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0033700029999999999, , 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:54.106 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_started", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.824625+0000", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.8889999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000211175 , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 3.8890000000000002e-05 , 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.630 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 37 , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:54.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144504+0000", , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.0872999999999996e-05 , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.149407+0000", , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.8856000000000001e-05 , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384053+0000", , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.2444999999999998e-05 , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 187 , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.374736+0000", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.4740000000000002e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384434+0000", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.2589000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.581520+0000", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000112135 Ignoring malparsed backtrace: 2023-03-30T00:59:54.777 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578853+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578887+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.630036+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.833+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 20=[]:{12=[10]} for object 3:6d0110e8:::smithi20399497-36:c , 2023-03-30T00:59:54.834+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) , 2023-03-30T00:59:54.862 DEBUG:teuthology.orchestra.run.smithi203:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.15.asok dump_ops_in_flight Ignoring malparsed backtrace: 2023-03-30T00:59:55.276 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.622353+0000", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001925162 Ignoring malparsed backtrace: 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.302 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.629137+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.636040+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:55.353 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.6981000000000001e-05 , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.00041520000000000001 , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.809116+0000", , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.8238999999999999e-05 , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.773072+0000", , 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000102753 , 2023-03-30T00:59:55.418 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.786664+0000", , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0077388860000000004 , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0076006499999999996, , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "started", Ignoring malparsed backtrace: 2023-03-30T00:59:55.455 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.827928+0000", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.1890000000000007e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:56.235 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops , 2023-03-30T00:59:56.266 ERROR:teuthology.orchestra.daemon.state:Failed to send signal 1: None , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-30T00:59:56.868+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{11=[11,10,f,d],2c=[2c,29,28,27,26,23,20,1e,1c]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:56.870+0000 7fca99663700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{d=[d],20=[20,1f,1e,1d,1c,1b,17],2c=[2c,29,28,27,26,23]} for object 3:ac746cfe:::smithi20399497-5:c , 2023-03-30T00:59:56.870+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:bbe97148:::smithi20399497-47:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:57.884 INFO:teuthology.orchestra.run.smithi035.stdout:osd.0: {} , 2023-03-30T00:59:57.885+0000 7f7e40f3f700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 2 (PID: 94700) UID: 0 , 2023-03-30T00:59:57.888 INFO:tasks.rados.rados.0.smithi203.stdout:368: writing smithi20399497-3 from 4521984 to 4636672 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_change": "24", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend": "0", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend_prenautilus": "0", Ignoring malparsed backtrace: 2023-03-30T00:59:58.075 INFO:teuthology.orchestra.run.smithi035.stdout: "write_tier": -1, , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "cache_mode": "none", , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "target_max_bytes": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_grade_decay_rate": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_search_last_n": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "grade_table": [], Ignoring malparsed backtrace: 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "up_thru": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "down_at": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "lost_at": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.130 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", Ignoring malparsed backtrace: 2023-03-30T00:59:58.146 INFO:teuthology.orchestra.run.smithi035.stdout: "addrvec": [ , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", Ignoring malparsed backtrace: 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "uuid": "32fd7d07-dd9a-476c-9fc4-da8e1f2855c0", , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "up": 1, , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "in": 1, Ignoring malparsed backtrace: 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: "cluster_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6807", Ignoring malparsed backtrace: 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addr": "172.21.15.35:6809/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_front_addr": "172.21.15.35:6807/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "state": [ Ignoring malparsed backtrace: 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.181:6816", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94712 Ignoring malparsed backtrace: 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.243 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.35:6823", Ignoring malparsed backtrace: 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_begin": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_end": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "up_from": 22, Ignoring malparsed backtrace: 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6810", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94976 Ignoring malparsed backtrace: 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "osd": 5, , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_interval": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout: "num_ops": 0 , 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout:} , 2023-03-30T00:59:58.312 INFO:tasks.thrashosds.thrasher:j is {'epoch': 86, 'fsid': '8f2049d8-2bb0-46fe-a814-ce26c43a7f40', 'created': '2023-03-30T00:58:33.472724+0000', 'modified': '2023-03-30T00:59:57.844713+0000', 'last_up_change': '2023-03-30T00:59:43.755794+0000', 'last_in_change': '2023-03-30T00:58:39.154518+0000', 'flags': 'sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit', 'flags_num': 5799936, 'flags_set': ['pglog_hardlimit', 'purged_snapdirs', 'recovery_deletes', 'sortbitwise'], 'crush_version': 7, 'full_ratio': 0.949999988079071, 'backfillfull_ratio': 0.8999999761581421, 'nearfull_ratio': 0.8500000238418579, 'cluster_snapshot': '', 'pool_max': 3, 'max_osd': 16, 'require_min_compat_client': 'luminous', 'min_compat_client': 'jewel', 'require_osd_release': 'reef', 'allow_crimson': False, 'pools': [{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-30T00:58:45.356303+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '24', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 15.380000114440918, 'score_stable': 15.380000114440918, 'optimal_score': 0.12999999523162842, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 2, 'pool_name': 'rbd', 'create_time': '2023-03-30T00:58:49.826143+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '27', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}, 'read_balance': {'score_acting': 3.7699999809265137, 'score_stable': 3.7699999809265137, 'optimal_score': 0.7300000190734863, 'raw_score_acting': 2.75, 'raw_score_stable': 2.75, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 3, 'pool_name': 'unique_pool_0', 'create_time': '2023-03-30T00:59:00.541501+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 3, 'size': 7, 'min_size': 5, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 36, 'pg_placement_num': 16, 'pg_placement_num_target': 36, 'pg_num_target': 36, 'pg_num_pending': 36, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '54', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 54, 'snap_epoch': 86, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'shecprofile', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 16384, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rados': {}}}], 'osds': [{'osd': 0, 'uuid': 'ebbb26ab-20a8-4846-8b3f-e2fbabb7cf75', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 42, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6810', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6811', 'nonce': 95086}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6812', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6813', 'nonce': 95086}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6816', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6817', 'nonce': 95086}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6814', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6815', 'nonce': 95086}]}, 'public_addr': '172.21.15.35:6811/95086', 'cluster_addr': '172.21.15.35:6813/95086', 'heartbeat_back_addr': '172.21.15.35:6817/95086', 'heartbeat_front_addr': '172.21.15.35:6815/95086', 'state': ['exists', 'up']}, {'osd': 1, 'uuid': '3bc34814-68b2-4d01-8249-953700168560', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6800', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6801', 'nonce': 94718}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6802', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6803', 'nonce': 94718}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6806', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6807', 'nonce': 94718}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6804', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6805', 'nonce': 94718}]}, 'public_addr': '172.21.15.83:6801/94718', 'cluster_addr': '172.21.15.83:6803/94718', 'heartbeat_back_addr': '172.21.15.83:6807/94718', 'heartbeat_front_addr': '172.21.15.83:6805/94718', 'state': ['exists', 'up']}, {'osd': 2, 'uuid': '76f71867-1d44-4527-877a-fd92aac57333', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6808', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6809', 'nonce': 94710}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6810', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6811', 'nonce': 94710}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6814', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6815', 'nonce': 94710}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6812', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6813', 'nonce': 94710}]}, 'public_addr': '172.21.15.181:6809/94710', 'cluster_addr': '172.21.15.181:6811/94710', 'heartbeat_back_addr': '172.21.15.181:6815/94710', 'heartbeat_front_addr': '172.21.15.181:6813/94710', 'state': ['exists', 'up']}, {'osd': 3, 'uuid': '954b6f1d-986f-49e6-93e5-d3423f3bf421', 'up': 0, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 22, 'last_clean_end': 27, 'up_from': 70, 'up_thru': 0, 'down_at': 72, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6816', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6817', 'nonce': 100463}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6818', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6819', 'nonce': 100463}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6822', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6823', 'nonce': 100463}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6820', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6821', 'nonce': 100463}]}, 'public_addr': '172.21.15.203:6817/100463', 'cluster_addr': '172.21.15.203:6819/100463', 'heartbeat_back_addr': '172.21.15.203:6823/100463', 'heartbeat_front_addr': '172.21.15.203:6821/100463', 'state': ['exists']}, {'osd': 4, 'uuid': 'ad653e95-e58a-4fec-9db8-d313042c2e97', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6826', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6827', 'nonce': 95091}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6828', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6829', 'nonce': 95091}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6832', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6833', 'nonce': 95091}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6830', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6831', 'nonce': 95091}]}, 'public_addr': '172.21.15.35:6827/95091', 'cluster_addr': '172.21.15.35:6829/95091', 'heartbeat_back_addr': '172.21.15.35:6833/95091', 'heartbeat_front_addr': '172.21.15.35:6831/95091', 'state': ['exists', 'up']}, {'osd': 5, 'uuid': '32fd7d07-dd9a-476c-9fc4-da8e1f2855c0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6808', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6809', 'nonce': 94717}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6810', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6811', 'nonce': 94717}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6814', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6815', 'nonce': 94717}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6812', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6813', 'nonce': 94717}]}, 'public_addr': '172.21.15.83:6809/94717', 'cluster_addr': '172.21.15.83:6811/94717', 'heartbeat_back_addr': '172.21.15.83:6815/94717', 'heartbeat_front_addr': '172.21.15.83:6813/94717', 'state': ['exists', 'up']}, {'osd': 6, 'uuid': '28a11dac-0c35-4a42-8250-cfedff301ba0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 0, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6800', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6801', 'nonce': 94711}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6802', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6803', 'nonce': 94711}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6806', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6807', 'nonce': 94711}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6804', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6805', 'nonce': 94711}]}, 'public_addr': '172.21.15.181:6801/94711', 'cluster_addr': '172.21.15.181:6803/94711', 'heartbeat_back_addr': '172.21.15.181:6807/94711', 'heartbeat_front_addr': '172.21.15.181:6805/94711', 'state': ['exists', 'up']}, {'osd': 7, 'uuid': 'ea60440a-7f79-4503-9ddb-b513c366c68a', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6800', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6801', 'nonce': 94975}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6802', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6803', 'nonce': 94975}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6806', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6807', 'nonce': 94975}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6804', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6805', 'nonce': 94975}]}, 'public_addr': '172.21.15.203:6801/94975', 'cluster_addr': '172.21.15.203:6803/94975', 'heartbeat_back_addr': '172.21.15.203:6807/94975', 'heartbeat_front_addr': '172.21.15.203:6805/94975', 'state': ['exists', 'up']}, {'osd': 8, 'uuid': '2614c844-ce70-4875-86cd-7ce40b5002db', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6802', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6803', 'nonce': 95087}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6804', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6805', 'nonce': 95087}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6808', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6809', 'nonce': 95087}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6806', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6807', 'nonce': 95087}]}, 'public_addr': '172.21.15.35:6803/95087', 'cluster_addr': '172.21.15.35:6805/95087', 'heartbeat_back_addr': '172.21.15.35:6809/95087', 'heartbeat_front_addr': '172.21.15.35:6807/95087', 'state': ['exists', 'up']}, {'osd': 9, 'uuid': '110531cf-e218-4332-aef0-9f05d71178bb', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6817', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6819', 'nonce': 94719}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6821', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6823', 'nonce': 94719}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6829', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6831', 'nonce': 94719}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6825', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6827', 'nonce': 94719}]}, 'public_addr': '172.21.15.83:6819/94719', 'cluster_addr': '172.21.15.83:6823/94719', 'heartbeat_back_addr': '172.21.15.83:6831/94719', 'heartbeat_front_addr': '172.21.15.83:6827/94719', 'state': ['exists', 'up']}, {'osd': 10, 'uuid': 'c7b567e6-1561-4f29-bbd4-65a6a4746929', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6816', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6817', 'nonce': 94712}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6818', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6819', 'nonce': 94712}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6825', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6827', 'nonce': 94712}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6821', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6823', 'nonce': 94712}]}, 'public_addr': '172.21.15.181:6817/94712', 'cluster_addr': '172.21.15.181:6819/94712', 'heartbeat_back_addr': '172.21.15.181:6827/94712', 'heartbeat_front_addr': '172.21.15.181:6823/94712', 'state': ['exists', 'up']}, {'osd': 11, 'uuid': '36e97302-d025-4aaf-ade4-3ece93f0ffd6', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6824', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6825', 'nonce': 94977}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6826', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6827', 'nonce': 94977}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6830', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6831', 'nonce': 94977}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6828', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6829', 'nonce': 94977}]}, 'public_addr': '172.21.15.203:6825/94977', 'cluster_addr': '172.21.15.203:6827/94977', 'heartbeat_back_addr': '172.21.15.203:6831/94977', 'heartbeat_front_addr': '172.21.15.203:6829/94977', 'state': ['exists', 'up']}, {'osd': 12, 'uuid': 'bb2a234d-18d7-4ed3-b250-573eeb31cc0b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6818', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6819', 'nonce': 95092}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6820', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6821', 'nonce': 95092}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6824', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6825', 'nonce': 95092}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6822', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6823', 'nonce': 95092}]}, 'public_addr': '172.21.15.35:6819/95092', 'cluster_addr': '172.21.15.35:6821/95092', 'heartbeat_back_addr': '172.21.15.35:6825/95092', 'heartbeat_front_addr': '172.21.15.35:6823/95092', 'state': ['exists', 'up']}, {'osd': 13, 'uuid': '34d7ae22-e68e-4faa-b647-b251e0b7336b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6816', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6818', 'nonce': 94720}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6820', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6822', 'nonce': 94720}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6828', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6830', 'nonce': 94720}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6824', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6826', 'nonce': 94720}]}, 'public_addr': '172.21.15.83:6818/94720', 'cluster_addr': '172.21.15.83:6822/94720', 'heartbeat_back_addr': '172.21.15.83:6830/94720', 'heartbeat_front_addr': '172.21.15.83:6826/94720', 'state': ['exists', 'up']}, {'osd': 14, 'uuid': '72801aa3-e45b-4a6f-9f52-d327b2caa7aa', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6820', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6822', 'nonce': 94713}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6824', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6826', 'nonce': 94713}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6830', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6831', 'nonce': 94713}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6828', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6829', 'nonce': 94713}]}, 'public_addr': '172.21.15.181:6822/94713', 'cluster_addr': '172.21.15.181:6826/94713', 'heartbeat_back_addr': '172.21.15.181:6831/94713', 'heartbeat_front_addr': '172.21.15.181:6829/94713', 'state': ['exists', 'up']}, {'osd': 15, 'uuid': '99ba406f-ecd6-46eb-bddb-66ae54f63dcf', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6808', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6809', 'nonce': 94976}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6810', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6811', 'nonce': 94976}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6814', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6815', 'nonce': 94976}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6812', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6813', 'nonce': 94976}]}, 'public_addr': '172.21.15.203:6809/94976', 'cluster_addr': '172.21.15.203:6811/94976', 'heartbeat_back_addr': '172.21.15.203:6815/94976', 'heartbeat_front_addr': '172.21.15.203:6813/94976', 'state': ['exists', 'up']}], 'osd_xinfo': [{'osd': 0, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 1, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 2, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 3, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 72}, {'osd': 4, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 5, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 6, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 7, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 8, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 9, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 10, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 11, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 12, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 13, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 14, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 15, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}], 'pg_upmap': [], 'pg_upmap_items': [], 'pg_upmap_primaries': [], 'pg_temp': [], 'primary_temp': [], 'blocklist': {}, 'range_blocklist': {}, 'erasure_code_profiles': {'default': {'crush-failure-domain': 'osd', 'k': '2', 'm': '1', 'plugin': 'jerasure', 'technique': 'reed_sol_van'}, 'shecprofile': {'c': '2', 'crush-device-class': '', 'crush-failure-domain': 'osd', 'crush-root': 'default', 'k': '4', 'm': '3', 'name': 'shecprofile', 'plugin': 'shec', 'technique': 'multiple'}}, 'removed_snaps_queue': [{'pool': 3, 'snaps': [{'begin': 2, 'length': 2}, {'begin': 5, 'length': 2}, {'begin': 10, 'length': 1}, {'begin': 12, 'length': 2}, {'begin': 15, 'length': 4}, {'begin': 23, 'length': 2}, {'begin': 27, 'length': 1}, {'begin': 29, 'length': 1}, {'begin': 31, 'length': 1}, {'begin': 34, 'length': 2}, {'begin': 39, 'length': 1}]}], 'new_removed_snaps': [], 'new_purged_snaps': [], 'crush_node_flags': {}, 'device_class_flags': {}, 'stretch_mode': {'stretch_mode_enabled': False, 'stretch_bucket_count': 0, 'degraded_stretch_mode': 0, 'recovering_stretch_mode': 0, 'stretch_mode_bucket': 0}} Ignoring malparsed backtrace: 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:2f snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {27, 28, 29, 2c, 2d, 2e, 2f} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:26 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {22, 23, 26} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:20 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {1e, 1f, 20} ...repaired Ignoring malparsed backtrace: 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.712749+0000", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.681262+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.1990999999999999e-05 , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.713837+0000", , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.0708e-05 , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.806010+0000", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.814067+0000", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.3025000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.858217+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:58.864 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd set noscrub , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:376: writing smithi20399497-48 from 2457600 to 2621440 tid 1 , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:377: rmattr oid 19 current snap is 33 Ignoring malparsed backtrace: 2023-03-30T00:59:59.089 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 16 v 50 (ObjNum 134 snap 26 seq_num 134) dirty exists , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 1 to smithi20399497-17 , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 2 to smithi20399497-17 Ignoring malparsed backtrace: 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0011711810000000001 , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "commit sent; apply or cleanup", , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "client_info": { Ignoring malparsed backtrace: 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606447+0000", , 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.3420000000000001e-06 , 2023-03-30T00:59:59.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.613875+0000", , 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0042258850000000004 , 2023-03-30T00:59:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.691580+0000", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999971 Ignoring malparsed backtrace: 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.688990+0000", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0050633730000000004 Ignoring malparsed backtrace: 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732738+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.9600000000000006e-06 , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001601318 , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.830761+0000", , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.569096+0000", , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.6s5 36/30 ECSubWrite(tid=17, reqid=client.4555.0:186, at_version=36'9, trim_to=0'0, roll_forward_to=33'6))", Ignoring malparsed backtrace: 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.598900+0000", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.8109999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.603403+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.4739000000000001e-05 , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732868+0000", , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.4030000000000007e-06 , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.034 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:38.760206+0000", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:00.047 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.845424+0000", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.2374999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.854031+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.214 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops , 2023-03-30T01:00:00.274+0000 7f6b68ab7700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 94966) UID: 0 , 2023-03-30T01:00:00.320 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false Ignoring malparsed backtrace: 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:175 3.as0 3:5d617aee:::smithi20399497-20:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e36)", , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:07.550221+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "tid": 388 , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T01:00:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606225+0000", , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.673419+0000", , 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.0645e-05 , 2023-03-30T01:00:00.530 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.703355+0000", , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 7.6669999999999996e-06 , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:705 3.fs0 3:f76751ac:::smithi20399497-42:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e67)", Ignoring malparsed backtrace: 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.800579+0000", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.00046025699999999999 Ignoring malparsed backtrace: 2023-03-30T01:00:00.580 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.805906+0000", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001011241 Ignoring malparsed backtrace: 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 17.667686744000001, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.014015517, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.808867+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "started", , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.811121+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.631 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } Ignoring malparsed backtrace: 2023-03-30T01:00:00.726 INFO:teuthology.orchestra.run.smithi035.stdout:osd.13: {} , 2023-03-30T01:00:00.746 INFO:teuthology.orchestra.run.smithi035.stdout:osd.14: {} , 2023-03-30T01:00:00.767 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T01:00:01.589+0000 7f5cfc24b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 10 (PID: 94706) UID: 0 , 2023-03-30T01:00:01.614 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument Ignoring malparsed backtrace: 2023-03-30T01:00:03.108 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} , 2023-03-30T01:00:03.128 INFO:teuthology.orchestra.run.smithi035.stdout:osd.11: {} , 2023-03-30T01:00:03.133 INFO:teuthology.orchestra.run.smithi203.stdout:{ Ignoring malparsed backtrace: 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.332416+0000", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1637999999999999e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.812171+0000", , 2023-03-30T01:00:03.480 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.3927e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.500 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.820021+0000", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 2.446e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T01:00:03.515 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 140 Ignoring malparsed backtrace: 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144542+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.542 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.381561+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384196+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_info": { , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", Ignoring malparsed backtrace: 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:03.611 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", Ignoring malparsed backtrace: 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.5550000000000002e-06 , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000104752 , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.705642+0000", , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.759581+0000", , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1364e-05 , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0038820090000000001 , 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.138 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", , 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:04.153 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.628576+0000", , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 7.4080000000000001e-06 , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.191 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.635982+0000", , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0049630509999999996 , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.721032+0000", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.724382+0000", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 9.4830000000000005e-06 Ignoring malparsed backtrace: 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.731143+0000", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001987801 Ignoring malparsed backtrace: 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.726052+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:33.738720+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.767733+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T01:00:04.342 INFO:teuthology.orchestra.run.smithi203.stdout: "description": "osd_op(client.4555.0:780 3.13s0 3:cbc5c355:::smithi20399497-3:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e72)", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "initiated_at": "2023-03-30T00:59:43.841834+0000", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "age": 20.282786238, Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x135) [0x55a59601f2ad] , 2: ceph-osd(+0x631473) [0x55a59601f473] Ignoring malparsed backtrace: 2023-03-30T00:59:43.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:58:48.149595+0000", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op must wait for map", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.575214+0000", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7829999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 26.474046659999999, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062746649999999996, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:16.606341+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_started", , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:23.708494+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:23.712400+0000", , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.367927426000001, , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076374399999999997, Ignoring malparsed backtrace: 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010535 , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:541 3.1es0 3:7ef085b3:::smithi20399497-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:26.715419+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703128+0000", , 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2414e-05 , 2023-03-30T00:59:43.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703333+0000", , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012205999999999999 , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.774152+0000", , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00084599299999999996 , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, , 2023-03-30T00:59:43.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, Ignoring malparsed backtrace: 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0399999999999998e-06 , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.585362+0000", , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.614174+0000", , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7903e-05 , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.678976+0000", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.683130+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.690158+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpWrite(3.as5 71/54 ECSubWrite(tid=203, reqid=client.4555.0:735, at_version=71'38, trim_to=0'0, roll_forward_to=71'35))", , 2023-03-30T00:59:43.801 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.774378+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.807553+0000", , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012399939999999999 , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.812386+0000", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 Ignoring malparsed backtrace: 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 43 v 48 (ObjNum 129 snap 24 seq_num 129) dirty exists , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:300: left oid 43 (ObjNum 129 snap 24 seq_num 129) , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:302: finishing write tid 1 to smithi20399497-3 Ignoring malparsed backtrace: 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.581882+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.587173+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068317159999999998 , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:18.626041+0000", , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.059e-05 , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.685149+0000", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.758157+0000", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068448800000000002 Ignoring malparsed backtrace: 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.809042+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.440 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", Ignoring malparsed backtrace: 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:44.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00063433800000000002 , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.617 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768302+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.773+0000 7f7e20ae9700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:756d2f96:::smithi20399497-24:c repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:c63dffb4:::smithi20399497-40:6 repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{12=[12,10,d]} for object 3:cb5d7287:::smithi20399497-32:c Ignoring malparsed backtrace: 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.600924+0000", , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000208836 , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:44.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665069+0000", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665141+0000", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2509999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:44.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703115+0000", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0409000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.743640+0000", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076826619999999998 Ignoring malparsed backtrace: 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.755311+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.766273+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.4555", , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", Ignoring malparsed backtrace: 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", Ignoring malparsed backtrace: 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9876999999999993e-05 , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.792080+0000", , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8390000000000003e-06 , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:744 3.10s0 3:0aab8796:::smithi20399497-28:head [read 0~0,checksum] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e71)", Ignoring malparsed backtrace: 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.814167+0000", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2109999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:45.731+0000 7efdb31a1700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 11 (PID: 94971) UID: 0 , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:307: done (2 left) , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:308: rollback oid 45 current snap is 25 Ignoring malparsed backtrace: 2023-03-30T00:59:46.791+0000 7f2881c66700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:ef8379c0:::smithi20399497-18:5 repair needed (no obc) , 2023-03-30T00:59:46.791+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{11=[11,10,f,d]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:46.789+0000 7f579d0a4700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:5d617aee:::smithi20399497-20:3 repair needed (no obc) Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T00:59:47.581 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true , 2023-03-30T00:59:47.650+0000 7f0f9634b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 95055) UID: 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.5651e-05 , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0048024890000000001 , 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.888 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.900 INFO:teuthology.orchestra.run.smithi035.stdout: "tid": 249 , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.605832+0000", , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0661e-05 , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.926 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.618433+0000", , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.2633999999999995e-05 , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707573+0000", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707643+0000", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.7330000000000003e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "reached_pg", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680627+0000", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010247900000000001 Ignoring malparsed backtrace: 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.690506+0000", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.9204999999999997e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705513+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "op must wait for map", , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705691+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.710594+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.717126+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:48.075 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} , 2023-03-30T00:59:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 , 2023-03-30T00:59:48.076 INFO:tasks.rados.rados.0.smithi203.stdout:324: writing smithi20399497-19 from 2293760 to 3047424 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "complaint_time": 30, , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "num_blocked_ops": 0 , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010960500000000001 , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:527 3.14s0 3:2e5a366f:::smithi20399497-31:head [append 0~0] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:26.695461+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.698204+0000", , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.704831+0000", , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 9.7558000000000003e-05 , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:534 3.4s0 3:24eec11e:::smithi20399497-7:head [setxattr _header (56) in=7b] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", Ignoring malparsed backtrace: 2023-03-30T00:59:48.646 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.705518+0000", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.691686+0000", , 2023-03-30T00:59:48.671 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.01079631 Ignoring malparsed backtrace: 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680770+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.698 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "started", , 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "client_info": { , 2023-03-30T00:59:48.712 INFO:teuthology.orchestra.run.smithi035.stdout: "client": "client.4555", Ignoring malparsed backtrace: 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.2860000000000002e-06 , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0061165480000000003 , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: } , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:48.833+0000 7efd99558700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:14c33e9f:::smithi20399497-34:3 repair needed (no obc) , 2023-03-30T00:59:48.836 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 0 to 802816 tid 1 , 2023-03-30T00:59:48.838 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 802816 to 1228800 tid 2 Ignoring malparsed backtrace: 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ , 2023-03-30T00:59:49.307 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0722e-05 , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.604113+0000", , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00052637799999999998 , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0021209829999999999, , 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "type_data": { , 2023-03-30T00:59:49.363 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.610251+0000", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.7749999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "all_read", , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.608123+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:17.611077+0000", , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "age": 31.691323830999998, , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0017984959999999999, Ignoring malparsed backtrace: 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.789051+0000", , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.796145+0000", , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0040419669999999996 , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.491 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.798914+0000", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.9406e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 523, in _write_all , count = self._write(data) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1364, in _write Ignoring malparsed backtrace: 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "initiated", , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.560616+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.583527+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.2229999999999996e-06 , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.683455+0000", , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.7909999999999999e-06 , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.680363+0000", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.687466+0000", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.7453e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:50.135 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.765837+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.1ds1 77/54 ECSubWrite(tid=239, reqid=client.4555.0:880, at_version=77'66, trim_to=0'0, roll_forward_to=77'64))", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:48.854994+0000", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 1.1714130300000001, Ignoring malparsed backtrace: 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.8422000000000001e-05 , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.486+0000 7f6a8da31700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 14 (PID: 94707) UID: 0 , 2023-03-30T00:59:50.492 INFO:teuthology.orchestra.run.smithi035.stdout:osd.9: {} , 2023-03-30T00:59:50.513 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} Ignoring malparsed backtrace: 2023-03-30T00:59:51.270 INFO:teuthology.orchestra.run.smithi083.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:51.271 INFO:teuthology.orchestra.run.smithi083.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. , 2023-03-30T00:59:51.280 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops Ignoring malparsed backtrace: 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 1 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 2 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:348: finishing copy_from to smithi20399497-15 Ignoring malparsed backtrace: 2023-03-30T00:59:53.011+0000 7f57b74ee700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 13 (PID: 94715) UID: 0 , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. Ignoring malparsed backtrace: 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.599544+0000", , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0852000000000005e-05 , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.684493+0000", , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121767 , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.040131732000000003, , 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { , 2023-03-30T00:59:53.381 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2023-03-30T00:59:53.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703140+0000", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.756116+0000", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105009 Ignoring malparsed backtrace: 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:53.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779826+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779943+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: ] , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0249999999999998e-06 , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpReadReply(3.9s0 71/42 ECSubReadReply(tid=160, attrs_read=0))", , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.803584+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768289+0000", , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0489999999999999e-06 , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.521 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 756 , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:48.845453+0000", , 2023-03-30T00:59:53.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.843+0000 7fc5039f5700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 8 (PID: 95060) UID: 0 , 2023-03-30T00:59:53.847 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 13 1 , 2023-03-30T00:59:53.884+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0117e-05 , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.654122+0000", , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.688971+0000", , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0037431610000000001 , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.015 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.710655+0000", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.6818000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.743870+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.041 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.754628+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0947e-05 , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.813138+0000", , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.9743999999999999e-05 , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0033700029999999999, , 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:54.106 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_started", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.824625+0000", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.8889999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000211175 , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 3.8890000000000002e-05 , 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.630 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 37 , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:54.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144504+0000", , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.0872999999999996e-05 , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.149407+0000", , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.8856000000000001e-05 , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384053+0000", , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.2444999999999998e-05 , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 187 , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.374736+0000", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.4740000000000002e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384434+0000", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.2589000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.581520+0000", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000112135 Ignoring malparsed backtrace: 2023-03-30T00:59:54.777 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578853+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578887+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.630036+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.833+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 20=[]:{12=[10]} for object 3:6d0110e8:::smithi20399497-36:c , 2023-03-30T00:59:54.834+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) , 2023-03-30T00:59:54.862 DEBUG:teuthology.orchestra.run.smithi203:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.15.asok dump_ops_in_flight Ignoring malparsed backtrace: 2023-03-30T00:59:55.276 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.622353+0000", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001925162 Ignoring malparsed backtrace: 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.302 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.629137+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.636040+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:55.353 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.6981000000000001e-05 , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.00041520000000000001 , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.809116+0000", , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.8238999999999999e-05 , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.773072+0000", , 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000102753 , 2023-03-30T00:59:55.418 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.786664+0000", , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0077388860000000004 , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0076006499999999996, , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "started", Ignoring malparsed backtrace: 2023-03-30T00:59:55.455 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.827928+0000", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.1890000000000007e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:56.235 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops , 2023-03-30T00:59:56.266 ERROR:teuthology.orchestra.daemon.state:Failed to send signal 1: None , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-30T00:59:56.868+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{11=[11,10,f,d],2c=[2c,29,28,27,26,23,20,1e,1c]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:56.870+0000 7fca99663700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{d=[d],20=[20,1f,1e,1d,1c,1b,17],2c=[2c,29,28,27,26,23]} for object 3:ac746cfe:::smithi20399497-5:c , 2023-03-30T00:59:56.870+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:bbe97148:::smithi20399497-47:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:57.884 INFO:teuthology.orchestra.run.smithi035.stdout:osd.0: {} , 2023-03-30T00:59:57.885+0000 7f7e40f3f700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 2 (PID: 94700) UID: 0 , 2023-03-30T00:59:57.888 INFO:tasks.rados.rados.0.smithi203.stdout:368: writing smithi20399497-3 from 4521984 to 4636672 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_change": "24", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend": "0", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend_prenautilus": "0", Ignoring malparsed backtrace: 2023-03-30T00:59:58.075 INFO:teuthology.orchestra.run.smithi035.stdout: "write_tier": -1, , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "cache_mode": "none", , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "target_max_bytes": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_grade_decay_rate": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_search_last_n": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "grade_table": [], Ignoring malparsed backtrace: 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "up_thru": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "down_at": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "lost_at": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.130 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", Ignoring malparsed backtrace: 2023-03-30T00:59:58.146 INFO:teuthology.orchestra.run.smithi035.stdout: "addrvec": [ , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", Ignoring malparsed backtrace: 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "uuid": "32fd7d07-dd9a-476c-9fc4-da8e1f2855c0", , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "up": 1, , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "in": 1, Ignoring malparsed backtrace: 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: "cluster_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6807", Ignoring malparsed backtrace: 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addr": "172.21.15.35:6809/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_front_addr": "172.21.15.35:6807/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "state": [ Ignoring malparsed backtrace: 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.181:6816", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94712 Ignoring malparsed backtrace: 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.243 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.35:6823", Ignoring malparsed backtrace: 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_begin": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_end": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "up_from": 22, Ignoring malparsed backtrace: 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6810", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94976 Ignoring malparsed backtrace: 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "osd": 5, , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_interval": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout: "num_ops": 0 , 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout:} , 2023-03-30T00:59:58.312 INFO:tasks.thrashosds.thrasher:j is {'epoch': 86, 'fsid': '8f2049d8-2bb0-46fe-a814-ce26c43a7f40', 'created': '2023-03-30T00:58:33.472724+0000', 'modified': '2023-03-30T00:59:57.844713+0000', 'last_up_change': '2023-03-30T00:59:43.755794+0000', 'last_in_change': '2023-03-30T00:58:39.154518+0000', 'flags': 'sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit', 'flags_num': 5799936, 'flags_set': ['pglog_hardlimit', 'purged_snapdirs', 'recovery_deletes', 'sortbitwise'], 'crush_version': 7, 'full_ratio': 0.949999988079071, 'backfillfull_ratio': 0.8999999761581421, 'nearfull_ratio': 0.8500000238418579, 'cluster_snapshot': '', 'pool_max': 3, 'max_osd': 16, 'require_min_compat_client': 'luminous', 'min_compat_client': 'jewel', 'require_osd_release': 'reef', 'allow_crimson': False, 'pools': [{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-30T00:58:45.356303+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '24', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 15.380000114440918, 'score_stable': 15.380000114440918, 'optimal_score': 0.12999999523162842, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 2, 'pool_name': 'rbd', 'create_time': '2023-03-30T00:58:49.826143+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '27', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}, 'read_balance': {'score_acting': 3.7699999809265137, 'score_stable': 3.7699999809265137, 'optimal_score': 0.7300000190734863, 'raw_score_acting': 2.75, 'raw_score_stable': 2.75, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 3, 'pool_name': 'unique_pool_0', 'create_time': '2023-03-30T00:59:00.541501+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 3, 'size': 7, 'min_size': 5, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 36, 'pg_placement_num': 16, 'pg_placement_num_target': 36, 'pg_num_target': 36, 'pg_num_pending': 36, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '54', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 54, 'snap_epoch': 86, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'shecprofile', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 16384, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rados': {}}}], 'osds': [{'osd': 0, 'uuid': 'ebbb26ab-20a8-4846-8b3f-e2fbabb7cf75', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 42, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6810', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6811', 'nonce': 95086}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6812', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6813', 'nonce': 95086}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6816', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6817', 'nonce': 95086}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6814', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6815', 'nonce': 95086}]}, 'public_addr': '172.21.15.35:6811/95086', 'cluster_addr': '172.21.15.35:6813/95086', 'heartbeat_back_addr': '172.21.15.35:6817/95086', 'heartbeat_front_addr': '172.21.15.35:6815/95086', 'state': ['exists', 'up']}, {'osd': 1, 'uuid': '3bc34814-68b2-4d01-8249-953700168560', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6800', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6801', 'nonce': 94718}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6802', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6803', 'nonce': 94718}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6806', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6807', 'nonce': 94718}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6804', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6805', 'nonce': 94718}]}, 'public_addr': '172.21.15.83:6801/94718', 'cluster_addr': '172.21.15.83:6803/94718', 'heartbeat_back_addr': '172.21.15.83:6807/94718', 'heartbeat_front_addr': '172.21.15.83:6805/94718', 'state': ['exists', 'up']}, {'osd': 2, 'uuid': '76f71867-1d44-4527-877a-fd92aac57333', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6808', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6809', 'nonce': 94710}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6810', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6811', 'nonce': 94710}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6814', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6815', 'nonce': 94710}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6812', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6813', 'nonce': 94710}]}, 'public_addr': '172.21.15.181:6809/94710', 'cluster_addr': '172.21.15.181:6811/94710', 'heartbeat_back_addr': '172.21.15.181:6815/94710', 'heartbeat_front_addr': '172.21.15.181:6813/94710', 'state': ['exists', 'up']}, {'osd': 3, 'uuid': '954b6f1d-986f-49e6-93e5-d3423f3bf421', 'up': 0, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 22, 'last_clean_end': 27, 'up_from': 70, 'up_thru': 0, 'down_at': 72, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6816', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6817', 'nonce': 100463}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6818', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6819', 'nonce': 100463}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6822', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6823', 'nonce': 100463}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6820', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6821', 'nonce': 100463}]}, 'public_addr': '172.21.15.203:6817/100463', 'cluster_addr': '172.21.15.203:6819/100463', 'heartbeat_back_addr': '172.21.15.203:6823/100463', 'heartbeat_front_addr': '172.21.15.203:6821/100463', 'state': ['exists']}, {'osd': 4, 'uuid': 'ad653e95-e58a-4fec-9db8-d313042c2e97', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6826', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6827', 'nonce': 95091}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6828', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6829', 'nonce': 95091}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6832', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6833', 'nonce': 95091}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6830', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6831', 'nonce': 95091}]}, 'public_addr': '172.21.15.35:6827/95091', 'cluster_addr': '172.21.15.35:6829/95091', 'heartbeat_back_addr': '172.21.15.35:6833/95091', 'heartbeat_front_addr': '172.21.15.35:6831/95091', 'state': ['exists', 'up']}, {'osd': 5, 'uuid': '32fd7d07-dd9a-476c-9fc4-da8e1f2855c0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6808', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6809', 'nonce': 94717}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6810', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6811', 'nonce': 94717}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6814', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6815', 'nonce': 94717}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6812', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6813', 'nonce': 94717}]}, 'public_addr': '172.21.15.83:6809/94717', 'cluster_addr': '172.21.15.83:6811/94717', 'heartbeat_back_addr': '172.21.15.83:6815/94717', 'heartbeat_front_addr': '172.21.15.83:6813/94717', 'state': ['exists', 'up']}, {'osd': 6, 'uuid': '28a11dac-0c35-4a42-8250-cfedff301ba0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 0, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6800', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6801', 'nonce': 94711}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6802', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6803', 'nonce': 94711}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6806', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6807', 'nonce': 94711}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6804', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6805', 'nonce': 94711}]}, 'public_addr': '172.21.15.181:6801/94711', 'cluster_addr': '172.21.15.181:6803/94711', 'heartbeat_back_addr': '172.21.15.181:6807/94711', 'heartbeat_front_addr': '172.21.15.181:6805/94711', 'state': ['exists', 'up']}, {'osd': 7, 'uuid': 'ea60440a-7f79-4503-9ddb-b513c366c68a', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6800', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6801', 'nonce': 94975}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6802', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6803', 'nonce': 94975}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6806', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6807', 'nonce': 94975}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6804', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6805', 'nonce': 94975}]}, 'public_addr': '172.21.15.203:6801/94975', 'cluster_addr': '172.21.15.203:6803/94975', 'heartbeat_back_addr': '172.21.15.203:6807/94975', 'heartbeat_front_addr': '172.21.15.203:6805/94975', 'state': ['exists', 'up']}, {'osd': 8, 'uuid': '2614c844-ce70-4875-86cd-7ce40b5002db', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6802', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6803', 'nonce': 95087}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6804', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6805', 'nonce': 95087}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6808', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6809', 'nonce': 95087}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6806', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6807', 'nonce': 95087}]}, 'public_addr': '172.21.15.35:6803/95087', 'cluster_addr': '172.21.15.35:6805/95087', 'heartbeat_back_addr': '172.21.15.35:6809/95087', 'heartbeat_front_addr': '172.21.15.35:6807/95087', 'state': ['exists', 'up']}, {'osd': 9, 'uuid': '110531cf-e218-4332-aef0-9f05d71178bb', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6817', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6819', 'nonce': 94719}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6821', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6823', 'nonce': 94719}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6829', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6831', 'nonce': 94719}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6825', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6827', 'nonce': 94719}]}, 'public_addr': '172.21.15.83:6819/94719', 'cluster_addr': '172.21.15.83:6823/94719', 'heartbeat_back_addr': '172.21.15.83:6831/94719', 'heartbeat_front_addr': '172.21.15.83:6827/94719', 'state': ['exists', 'up']}, {'osd': 10, 'uuid': 'c7b567e6-1561-4f29-bbd4-65a6a4746929', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6816', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6817', 'nonce': 94712}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6818', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6819', 'nonce': 94712}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6825', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6827', 'nonce': 94712}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6821', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6823', 'nonce': 94712}]}, 'public_addr': '172.21.15.181:6817/94712', 'cluster_addr': '172.21.15.181:6819/94712', 'heartbeat_back_addr': '172.21.15.181:6827/94712', 'heartbeat_front_addr': '172.21.15.181:6823/94712', 'state': ['exists', 'up']}, {'osd': 11, 'uuid': '36e97302-d025-4aaf-ade4-3ece93f0ffd6', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6824', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6825', 'nonce': 94977}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6826', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6827', 'nonce': 94977}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6830', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6831', 'nonce': 94977}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6828', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6829', 'nonce': 94977}]}, 'public_addr': '172.21.15.203:6825/94977', 'cluster_addr': '172.21.15.203:6827/94977', 'heartbeat_back_addr': '172.21.15.203:6831/94977', 'heartbeat_front_addr': '172.21.15.203:6829/94977', 'state': ['exists', 'up']}, {'osd': 12, 'uuid': 'bb2a234d-18d7-4ed3-b250-573eeb31cc0b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6818', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6819', 'nonce': 95092}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6820', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6821', 'nonce': 95092}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6824', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6825', 'nonce': 95092}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6822', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6823', 'nonce': 95092}]}, 'public_addr': '172.21.15.35:6819/95092', 'cluster_addr': '172.21.15.35:6821/95092', 'heartbeat_back_addr': '172.21.15.35:6825/95092', 'heartbeat_front_addr': '172.21.15.35:6823/95092', 'state': ['exists', 'up']}, {'osd': 13, 'uuid': '34d7ae22-e68e-4faa-b647-b251e0b7336b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6816', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6818', 'nonce': 94720}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6820', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6822', 'nonce': 94720}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6828', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6830', 'nonce': 94720}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6824', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6826', 'nonce': 94720}]}, 'public_addr': '172.21.15.83:6818/94720', 'cluster_addr': '172.21.15.83:6822/94720', 'heartbeat_back_addr': '172.21.15.83:6830/94720', 'heartbeat_front_addr': '172.21.15.83:6826/94720', 'state': ['exists', 'up']}, {'osd': 14, 'uuid': '72801aa3-e45b-4a6f-9f52-d327b2caa7aa', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6820', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6822', 'nonce': 94713}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6824', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6826', 'nonce': 94713}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6830', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6831', 'nonce': 94713}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6828', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6829', 'nonce': 94713}]}, 'public_addr': '172.21.15.181:6822/94713', 'cluster_addr': '172.21.15.181:6826/94713', 'heartbeat_back_addr': '172.21.15.181:6831/94713', 'heartbeat_front_addr': '172.21.15.181:6829/94713', 'state': ['exists', 'up']}, {'osd': 15, 'uuid': '99ba406f-ecd6-46eb-bddb-66ae54f63dcf', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6808', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6809', 'nonce': 94976}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6810', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6811', 'nonce': 94976}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6814', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6815', 'nonce': 94976}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6812', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6813', 'nonce': 94976}]}, 'public_addr': '172.21.15.203:6809/94976', 'cluster_addr': '172.21.15.203:6811/94976', 'heartbeat_back_addr': '172.21.15.203:6815/94976', 'heartbeat_front_addr': '172.21.15.203:6813/94976', 'state': ['exists', 'up']}], 'osd_xinfo': [{'osd': 0, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 1, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 2, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 3, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 72}, {'osd': 4, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 5, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 6, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 7, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 8, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 9, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 10, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 11, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 12, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 13, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 14, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 15, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}], 'pg_upmap': [], 'pg_upmap_items': [], 'pg_upmap_primaries': [], 'pg_temp': [], 'primary_temp': [], 'blocklist': {}, 'range_blocklist': {}, 'erasure_code_profiles': {'default': {'crush-failure-domain': 'osd', 'k': '2', 'm': '1', 'plugin': 'jerasure', 'technique': 'reed_sol_van'}, 'shecprofile': {'c': '2', 'crush-device-class': '', 'crush-failure-domain': 'osd', 'crush-root': 'default', 'k': '4', 'm': '3', 'name': 'shecprofile', 'plugin': 'shec', 'technique': 'multiple'}}, 'removed_snaps_queue': [{'pool': 3, 'snaps': [{'begin': 2, 'length': 2}, {'begin': 5, 'length': 2}, {'begin': 10, 'length': 1}, {'begin': 12, 'length': 2}, {'begin': 15, 'length': 4}, {'begin': 23, 'length': 2}, {'begin': 27, 'length': 1}, {'begin': 29, 'length': 1}, {'begin': 31, 'length': 1}, {'begin': 34, 'length': 2}, {'begin': 39, 'length': 1}]}], 'new_removed_snaps': [], 'new_purged_snaps': [], 'crush_node_flags': {}, 'device_class_flags': {}, 'stretch_mode': {'stretch_mode_enabled': False, 'stretch_bucket_count': 0, 'degraded_stretch_mode': 0, 'recovering_stretch_mode': 0, 'stretch_mode_bucket': 0}} Ignoring malparsed backtrace: 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:2f snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {27, 28, 29, 2c, 2d, 2e, 2f} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:26 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {22, 23, 26} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:20 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {1e, 1f, 20} ...repaired Ignoring malparsed backtrace: 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.712749+0000", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.681262+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.1990999999999999e-05 , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.713837+0000", , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.0708e-05 , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.806010+0000", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.814067+0000", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.3025000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.858217+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:58.864 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd set noscrub , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:376: writing smithi20399497-48 from 2457600 to 2621440 tid 1 , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:377: rmattr oid 19 current snap is 33 Ignoring malparsed backtrace: 2023-03-30T00:59:59.089 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 16 v 50 (ObjNum 134 snap 26 seq_num 134) dirty exists , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 1 to smithi20399497-17 , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 2 to smithi20399497-17 Ignoring malparsed backtrace: 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0011711810000000001 , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "commit sent; apply or cleanup", , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "client_info": { Ignoring malparsed backtrace: 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606447+0000", , 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.3420000000000001e-06 , 2023-03-30T00:59:59.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.613875+0000", , 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0042258850000000004 , 2023-03-30T00:59:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.691580+0000", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999971 Ignoring malparsed backtrace: 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.688990+0000", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0050633730000000004 Ignoring malparsed backtrace: 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732738+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.9600000000000006e-06 , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001601318 , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.830761+0000", , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.569096+0000", , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.6s5 36/30 ECSubWrite(tid=17, reqid=client.4555.0:186, at_version=36'9, trim_to=0'0, roll_forward_to=33'6))", Ignoring malparsed backtrace: 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.598900+0000", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.8109999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.603403+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.4739000000000001e-05 , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732868+0000", , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.4030000000000007e-06 , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.034 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:38.760206+0000", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:00.047 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.845424+0000", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.2374999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.854031+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.214 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops , 2023-03-30T01:00:00.274+0000 7f6b68ab7700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 94966) UID: 0 , 2023-03-30T01:00:00.320 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false Ignoring malparsed backtrace: 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:175 3.as0 3:5d617aee:::smithi20399497-20:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e36)", , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:07.550221+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "tid": 388 , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T01:00:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606225+0000", , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.673419+0000", , 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.0645e-05 , 2023-03-30T01:00:00.530 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.703355+0000", , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 7.6669999999999996e-06 , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:705 3.fs0 3:f76751ac:::smithi20399497-42:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e67)", Ignoring malparsed backtrace: 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.800579+0000", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.00046025699999999999 Ignoring malparsed backtrace: 2023-03-30T01:00:00.580 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.805906+0000", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001011241 Ignoring malparsed backtrace: 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 17.667686744000001, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.014015517, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.808867+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "started", , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.811121+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.631 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } Ignoring malparsed backtrace: 2023-03-30T01:00:00.726 INFO:teuthology.orchestra.run.smithi035.stdout:osd.13: {} , 2023-03-30T01:00:00.746 INFO:teuthology.orchestra.run.smithi035.stdout:osd.14: {} , 2023-03-30T01:00:00.767 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T01:00:01.589+0000 7f5cfc24b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 10 (PID: 94706) UID: 0 , 2023-03-30T01:00:01.614 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument Ignoring malparsed backtrace: 2023-03-30T01:00:03.108 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} , 2023-03-30T01:00:03.128 INFO:teuthology.orchestra.run.smithi035.stdout:osd.11: {} , 2023-03-30T01:00:03.133 INFO:teuthology.orchestra.run.smithi203.stdout:{ Ignoring malparsed backtrace: 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.332416+0000", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1637999999999999e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.812171+0000", , 2023-03-30T01:00:03.480 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.3927e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.500 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.820021+0000", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 2.446e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T01:00:03.515 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 140 Ignoring malparsed backtrace: 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144542+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.542 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.381561+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384196+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_info": { , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", Ignoring malparsed backtrace: 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:03.611 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", Ignoring malparsed backtrace: 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.5550000000000002e-06 , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000104752 , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.705642+0000", , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.759581+0000", , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1364e-05 , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0038820090000000001 , 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.138 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", , 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:04.153 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.628576+0000", , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 7.4080000000000001e-06 , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.191 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.635982+0000", , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0049630509999999996 , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.721032+0000", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.724382+0000", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 9.4830000000000005e-06 Ignoring malparsed backtrace: 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.731143+0000", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001987801 Ignoring malparsed backtrace: 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.726052+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:33.738720+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.767733+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T01:00:04.342 INFO:teuthology.orchestra.run.smithi203.stdout: "description": "osd_op(client.4555.0:780 3.13s0 3:cbc5c355:::smithi20399497-3:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e72)", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "initiated_at": "2023-03-30T00:59:43.841834+0000", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "age": 20.282786238, Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x135) [0x55a59601f2ad] , 2: ceph-osd(+0x631473) [0x55a59601f473] Ignoring malparsed backtrace: 2023-03-30T00:59:43.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:58:48.149595+0000", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op must wait for map", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.575214+0000", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7829999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 26.474046659999999, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062746649999999996, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:16.606341+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_started", , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:23.708494+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:23.712400+0000", , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.367927426000001, , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076374399999999997, Ignoring malparsed backtrace: 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010535 , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:541 3.1es0 3:7ef085b3:::smithi20399497-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:26.715419+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703128+0000", , 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2414e-05 , 2023-03-30T00:59:43.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703333+0000", , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012205999999999999 , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.774152+0000", , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00084599299999999996 , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, , 2023-03-30T00:59:43.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, Ignoring malparsed backtrace: 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0399999999999998e-06 , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.585362+0000", , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.614174+0000", , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7903e-05 , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.678976+0000", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.683130+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.690158+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpWrite(3.as5 71/54 ECSubWrite(tid=203, reqid=client.4555.0:735, at_version=71'38, trim_to=0'0, roll_forward_to=71'35))", , 2023-03-30T00:59:43.801 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.774378+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.807553+0000", , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012399939999999999 , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.812386+0000", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 Ignoring malparsed backtrace: 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 43 v 48 (ObjNum 129 snap 24 seq_num 129) dirty exists , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:300: left oid 43 (ObjNum 129 snap 24 seq_num 129) , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:302: finishing write tid 1 to smithi20399497-3 Ignoring malparsed backtrace: 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.581882+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.587173+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068317159999999998 , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:18.626041+0000", , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.059e-05 , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.685149+0000", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.758157+0000", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068448800000000002 Ignoring malparsed backtrace: 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.809042+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.440 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", Ignoring malparsed backtrace: 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:44.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00063433800000000002 , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.617 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768302+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.773+0000 7f7e20ae9700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:756d2f96:::smithi20399497-24:c repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:c63dffb4:::smithi20399497-40:6 repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{12=[12,10,d]} for object 3:cb5d7287:::smithi20399497-32:c Ignoring malparsed backtrace: 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.600924+0000", , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000208836 , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:44.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665069+0000", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665141+0000", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2509999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:44.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703115+0000", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0409000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.743640+0000", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076826619999999998 Ignoring malparsed backtrace: 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.755311+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.766273+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.4555", , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", Ignoring malparsed backtrace: 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", Ignoring malparsed backtrace: 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9876999999999993e-05 , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.792080+0000", , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8390000000000003e-06 , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:744 3.10s0 3:0aab8796:::smithi20399497-28:head [read 0~0,checksum] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e71)", Ignoring malparsed backtrace: 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.814167+0000", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2109999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:45.731+0000 7efdb31a1700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 11 (PID: 94971) UID: 0 , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:307: done (2 left) , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:308: rollback oid 45 current snap is 25 Ignoring malparsed backtrace: 2023-03-30T00:59:46.791+0000 7f2881c66700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:ef8379c0:::smithi20399497-18:5 repair needed (no obc) , 2023-03-30T00:59:46.791+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{11=[11,10,f,d]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:46.789+0000 7f579d0a4700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:5d617aee:::smithi20399497-20:3 repair needed (no obc) Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T00:59:47.581 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true , 2023-03-30T00:59:47.650+0000 7f0f9634b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 95055) UID: 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.5651e-05 , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0048024890000000001 , 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.888 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.900 INFO:teuthology.orchestra.run.smithi035.stdout: "tid": 249 , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.605832+0000", , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0661e-05 , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.926 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.618433+0000", , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.2633999999999995e-05 , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707573+0000", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707643+0000", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.7330000000000003e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "reached_pg", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680627+0000", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010247900000000001 Ignoring malparsed backtrace: 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.690506+0000", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.9204999999999997e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705513+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "op must wait for map", , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705691+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.710594+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.717126+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:48.075 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} , 2023-03-30T00:59:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 , 2023-03-30T00:59:48.076 INFO:tasks.rados.rados.0.smithi203.stdout:324: writing smithi20399497-19 from 2293760 to 3047424 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "complaint_time": 30, , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "num_blocked_ops": 0 , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010960500000000001 , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:527 3.14s0 3:2e5a366f:::smithi20399497-31:head [append 0~0] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:26.695461+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.698204+0000", , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.704831+0000", , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 9.7558000000000003e-05 , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:534 3.4s0 3:24eec11e:::smithi20399497-7:head [setxattr _header (56) in=7b] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", Ignoring malparsed backtrace: 2023-03-30T00:59:48.646 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.705518+0000", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.691686+0000", , 2023-03-30T00:59:48.671 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.01079631 Ignoring malparsed backtrace: 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680770+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.698 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "started", , 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "client_info": { , 2023-03-30T00:59:48.712 INFO:teuthology.orchestra.run.smithi035.stdout: "client": "client.4555", Ignoring malparsed backtrace: 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.2860000000000002e-06 , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0061165480000000003 , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: } , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:48.833+0000 7efd99558700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:14c33e9f:::smithi20399497-34:3 repair needed (no obc) , 2023-03-30T00:59:48.836 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 0 to 802816 tid 1 , 2023-03-30T00:59:48.838 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 802816 to 1228800 tid 2 Ignoring malparsed backtrace: 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ , 2023-03-30T00:59:49.307 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0722e-05 , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.604113+0000", , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00052637799999999998 , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0021209829999999999, , 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "type_data": { , 2023-03-30T00:59:49.363 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.610251+0000", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.7749999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "all_read", , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.608123+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:17.611077+0000", , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "age": 31.691323830999998, , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0017984959999999999, Ignoring malparsed backtrace: 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.789051+0000", , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.796145+0000", , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0040419669999999996 , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.491 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.798914+0000", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.9406e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 523, in _write_all , count = self._write(data) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1364, in _write Ignoring malparsed backtrace: 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "initiated", , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.560616+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.583527+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.2229999999999996e-06 , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.683455+0000", , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.7909999999999999e-06 , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.680363+0000", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.687466+0000", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.7453e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:50.135 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.765837+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.1ds1 77/54 ECSubWrite(tid=239, reqid=client.4555.0:880, at_version=77'66, trim_to=0'0, roll_forward_to=77'64))", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:48.854994+0000", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 1.1714130300000001, Ignoring malparsed backtrace: 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.8422000000000001e-05 , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.486+0000 7f6a8da31700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 14 (PID: 94707) UID: 0 , 2023-03-30T00:59:50.492 INFO:teuthology.orchestra.run.smithi035.stdout:osd.9: {} , 2023-03-30T00:59:50.513 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} Ignoring malparsed backtrace: 2023-03-30T00:59:51.270 INFO:teuthology.orchestra.run.smithi083.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:51.271 INFO:teuthology.orchestra.run.smithi083.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. , 2023-03-30T00:59:51.280 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops Ignoring malparsed backtrace: 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 1 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 2 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:348: finishing copy_from to smithi20399497-15 Ignoring malparsed backtrace: 2023-03-30T00:59:53.011+0000 7f57b74ee700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 13 (PID: 94715) UID: 0 , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. Ignoring malparsed backtrace: 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.599544+0000", , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0852000000000005e-05 , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.684493+0000", , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121767 , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.040131732000000003, , 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { , 2023-03-30T00:59:53.381 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2023-03-30T00:59:53.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703140+0000", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.756116+0000", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105009 Ignoring malparsed backtrace: 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:53.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779826+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779943+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: ] , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0249999999999998e-06 , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpReadReply(3.9s0 71/42 ECSubReadReply(tid=160, attrs_read=0))", , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.803584+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768289+0000", , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0489999999999999e-06 , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.521 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 756 , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:48.845453+0000", , 2023-03-30T00:59:53.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.843+0000 7fc5039f5700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 8 (PID: 95060) UID: 0 , 2023-03-30T00:59:53.847 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 13 1 , 2023-03-30T00:59:53.884+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0117e-05 , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.654122+0000", , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.688971+0000", , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0037431610000000001 , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.015 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.710655+0000", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.6818000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.743870+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.041 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.754628+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0947e-05 , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.813138+0000", , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.9743999999999999e-05 , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0033700029999999999, , 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:54.106 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_started", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.824625+0000", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.8889999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000211175 , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 3.8890000000000002e-05 , 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.630 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 37 , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:54.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144504+0000", , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.0872999999999996e-05 , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.149407+0000", , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.8856000000000001e-05 , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384053+0000", , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.2444999999999998e-05 , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 187 , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.374736+0000", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.4740000000000002e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384434+0000", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.2589000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.581520+0000", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000112135 Ignoring malparsed backtrace: 2023-03-30T00:59:54.777 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578853+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578887+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.630036+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.833+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 20=[]:{12=[10]} for object 3:6d0110e8:::smithi20399497-36:c , 2023-03-30T00:59:54.834+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) , 2023-03-30T00:59:54.862 DEBUG:teuthology.orchestra.run.smithi203:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.15.asok dump_ops_in_flight Ignoring malparsed backtrace: 2023-03-30T00:59:55.276 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.622353+0000", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001925162 Ignoring malparsed backtrace: 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.302 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.629137+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.636040+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:55.353 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.6981000000000001e-05 , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.00041520000000000001 , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.809116+0000", , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.8238999999999999e-05 , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.773072+0000", , 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000102753 , 2023-03-30T00:59:55.418 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.786664+0000", , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0077388860000000004 , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0076006499999999996, , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "started", Ignoring malparsed backtrace: 2023-03-30T00:59:55.455 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.827928+0000", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.1890000000000007e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:56.235 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops , 2023-03-30T00:59:56.266 ERROR:teuthology.orchestra.daemon.state:Failed to send signal 1: None , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-30T00:59:56.868+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{11=[11,10,f,d],2c=[2c,29,28,27,26,23,20,1e,1c]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:56.870+0000 7fca99663700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{d=[d],20=[20,1f,1e,1d,1c,1b,17],2c=[2c,29,28,27,26,23]} for object 3:ac746cfe:::smithi20399497-5:c , 2023-03-30T00:59:56.870+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:bbe97148:::smithi20399497-47:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:57.884 INFO:teuthology.orchestra.run.smithi035.stdout:osd.0: {} , 2023-03-30T00:59:57.885+0000 7f7e40f3f700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 2 (PID: 94700) UID: 0 , 2023-03-30T00:59:57.888 INFO:tasks.rados.rados.0.smithi203.stdout:368: writing smithi20399497-3 from 4521984 to 4636672 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_change": "24", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend": "0", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend_prenautilus": "0", Ignoring malparsed backtrace: 2023-03-30T00:59:58.075 INFO:teuthology.orchestra.run.smithi035.stdout: "write_tier": -1, , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "cache_mode": "none", , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "target_max_bytes": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_grade_decay_rate": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_search_last_n": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "grade_table": [], Ignoring malparsed backtrace: 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "up_thru": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "down_at": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "lost_at": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.130 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", Ignoring malparsed backtrace: 2023-03-30T00:59:58.146 INFO:teuthology.orchestra.run.smithi035.stdout: "addrvec": [ , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", Ignoring malparsed backtrace: 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "uuid": "32fd7d07-dd9a-476c-9fc4-da8e1f2855c0", , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "up": 1, , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "in": 1, Ignoring malparsed backtrace: 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: "cluster_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6807", Ignoring malparsed backtrace: 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addr": "172.21.15.35:6809/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_front_addr": "172.21.15.35:6807/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "state": [ Ignoring malparsed backtrace: 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.181:6816", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94712 Ignoring malparsed backtrace: 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.243 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.35:6823", Ignoring malparsed backtrace: 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_begin": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_end": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "up_from": 22, Ignoring malparsed backtrace: 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6810", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94976 Ignoring malparsed backtrace: 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "osd": 5, , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_interval": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout: "num_ops": 0 , 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout:} , 2023-03-30T00:59:58.312 INFO:tasks.thrashosds.thrasher:j is {'epoch': 86, 'fsid': '8f2049d8-2bb0-46fe-a814-ce26c43a7f40', 'created': '2023-03-30T00:58:33.472724+0000', 'modified': '2023-03-30T00:59:57.844713+0000', 'last_up_change': '2023-03-30T00:59:43.755794+0000', 'last_in_change': '2023-03-30T00:58:39.154518+0000', 'flags': 'sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit', 'flags_num': 5799936, 'flags_set': ['pglog_hardlimit', 'purged_snapdirs', 'recovery_deletes', 'sortbitwise'], 'crush_version': 7, 'full_ratio': 0.949999988079071, 'backfillfull_ratio': 0.8999999761581421, 'nearfull_ratio': 0.8500000238418579, 'cluster_snapshot': '', 'pool_max': 3, 'max_osd': 16, 'require_min_compat_client': 'luminous', 'min_compat_client': 'jewel', 'require_osd_release': 'reef', 'allow_crimson': False, 'pools': [{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-30T00:58:45.356303+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '24', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 15.380000114440918, 'score_stable': 15.380000114440918, 'optimal_score': 0.12999999523162842, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 2, 'pool_name': 'rbd', 'create_time': '2023-03-30T00:58:49.826143+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '27', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}, 'read_balance': {'score_acting': 3.7699999809265137, 'score_stable': 3.7699999809265137, 'optimal_score': 0.7300000190734863, 'raw_score_acting': 2.75, 'raw_score_stable': 2.75, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 3, 'pool_name': 'unique_pool_0', 'create_time': '2023-03-30T00:59:00.541501+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 3, 'size': 7, 'min_size': 5, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 36, 'pg_placement_num': 16, 'pg_placement_num_target': 36, 'pg_num_target': 36, 'pg_num_pending': 36, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '54', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 54, 'snap_epoch': 86, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'shecprofile', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 16384, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rados': {}}}], 'osds': [{'osd': 0, 'uuid': 'ebbb26ab-20a8-4846-8b3f-e2fbabb7cf75', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 42, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6810', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6811', 'nonce': 95086}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6812', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6813', 'nonce': 95086}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6816', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6817', 'nonce': 95086}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6814', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6815', 'nonce': 95086}]}, 'public_addr': '172.21.15.35:6811/95086', 'cluster_addr': '172.21.15.35:6813/95086', 'heartbeat_back_addr': '172.21.15.35:6817/95086', 'heartbeat_front_addr': '172.21.15.35:6815/95086', 'state': ['exists', 'up']}, {'osd': 1, 'uuid': '3bc34814-68b2-4d01-8249-953700168560', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6800', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6801', 'nonce': 94718}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6802', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6803', 'nonce': 94718}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6806', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6807', 'nonce': 94718}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6804', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6805', 'nonce': 94718}]}, 'public_addr': '172.21.15.83:6801/94718', 'cluster_addr': '172.21.15.83:6803/94718', 'heartbeat_back_addr': '172.21.15.83:6807/94718', 'heartbeat_front_addr': '172.21.15.83:6805/94718', 'state': ['exists', 'up']}, {'osd': 2, 'uuid': '76f71867-1d44-4527-877a-fd92aac57333', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6808', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6809', 'nonce': 94710}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6810', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6811', 'nonce': 94710}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6814', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6815', 'nonce': 94710}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6812', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6813', 'nonce': 94710}]}, 'public_addr': '172.21.15.181:6809/94710', 'cluster_addr': '172.21.15.181:6811/94710', 'heartbeat_back_addr': '172.21.15.181:6815/94710', 'heartbeat_front_addr': '172.21.15.181:6813/94710', 'state': ['exists', 'up']}, {'osd': 3, 'uuid': '954b6f1d-986f-49e6-93e5-d3423f3bf421', 'up': 0, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 22, 'last_clean_end': 27, 'up_from': 70, 'up_thru': 0, 'down_at': 72, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6816', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6817', 'nonce': 100463}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6818', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6819', 'nonce': 100463}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6822', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6823', 'nonce': 100463}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6820', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6821', 'nonce': 100463}]}, 'public_addr': '172.21.15.203:6817/100463', 'cluster_addr': '172.21.15.203:6819/100463', 'heartbeat_back_addr': '172.21.15.203:6823/100463', 'heartbeat_front_addr': '172.21.15.203:6821/100463', 'state': ['exists']}, {'osd': 4, 'uuid': 'ad653e95-e58a-4fec-9db8-d313042c2e97', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6826', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6827', 'nonce': 95091}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6828', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6829', 'nonce': 95091}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6832', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6833', 'nonce': 95091}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6830', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6831', 'nonce': 95091}]}, 'public_addr': '172.21.15.35:6827/95091', 'cluster_addr': '172.21.15.35:6829/95091', 'heartbeat_back_addr': '172.21.15.35:6833/95091', 'heartbeat_front_addr': '172.21.15.35:6831/95091', 'state': ['exists', 'up']}, {'osd': 5, 'uuid': '32fd7d07-dd9a-476c-9fc4-da8e1f2855c0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6808', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6809', 'nonce': 94717}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6810', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6811', 'nonce': 94717}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6814', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6815', 'nonce': 94717}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6812', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6813', 'nonce': 94717}]}, 'public_addr': '172.21.15.83:6809/94717', 'cluster_addr': '172.21.15.83:6811/94717', 'heartbeat_back_addr': '172.21.15.83:6815/94717', 'heartbeat_front_addr': '172.21.15.83:6813/94717', 'state': ['exists', 'up']}, {'osd': 6, 'uuid': '28a11dac-0c35-4a42-8250-cfedff301ba0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 0, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6800', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6801', 'nonce': 94711}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6802', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6803', 'nonce': 94711}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6806', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6807', 'nonce': 94711}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6804', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6805', 'nonce': 94711}]}, 'public_addr': '172.21.15.181:6801/94711', 'cluster_addr': '172.21.15.181:6803/94711', 'heartbeat_back_addr': '172.21.15.181:6807/94711', 'heartbeat_front_addr': '172.21.15.181:6805/94711', 'state': ['exists', 'up']}, {'osd': 7, 'uuid': 'ea60440a-7f79-4503-9ddb-b513c366c68a', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6800', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6801', 'nonce': 94975}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6802', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6803', 'nonce': 94975}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6806', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6807', 'nonce': 94975}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6804', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6805', 'nonce': 94975}]}, 'public_addr': '172.21.15.203:6801/94975', 'cluster_addr': '172.21.15.203:6803/94975', 'heartbeat_back_addr': '172.21.15.203:6807/94975', 'heartbeat_front_addr': '172.21.15.203:6805/94975', 'state': ['exists', 'up']}, {'osd': 8, 'uuid': '2614c844-ce70-4875-86cd-7ce40b5002db', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6802', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6803', 'nonce': 95087}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6804', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6805', 'nonce': 95087}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6808', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6809', 'nonce': 95087}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6806', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6807', 'nonce': 95087}]}, 'public_addr': '172.21.15.35:6803/95087', 'cluster_addr': '172.21.15.35:6805/95087', 'heartbeat_back_addr': '172.21.15.35:6809/95087', 'heartbeat_front_addr': '172.21.15.35:6807/95087', 'state': ['exists', 'up']}, {'osd': 9, 'uuid': '110531cf-e218-4332-aef0-9f05d71178bb', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6817', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6819', 'nonce': 94719}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6821', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6823', 'nonce': 94719}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6829', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6831', 'nonce': 94719}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6825', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6827', 'nonce': 94719}]}, 'public_addr': '172.21.15.83:6819/94719', 'cluster_addr': '172.21.15.83:6823/94719', 'heartbeat_back_addr': '172.21.15.83:6831/94719', 'heartbeat_front_addr': '172.21.15.83:6827/94719', 'state': ['exists', 'up']}, {'osd': 10, 'uuid': 'c7b567e6-1561-4f29-bbd4-65a6a4746929', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6816', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6817', 'nonce': 94712}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6818', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6819', 'nonce': 94712}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6825', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6827', 'nonce': 94712}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6821', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6823', 'nonce': 94712}]}, 'public_addr': '172.21.15.181:6817/94712', 'cluster_addr': '172.21.15.181:6819/94712', 'heartbeat_back_addr': '172.21.15.181:6827/94712', 'heartbeat_front_addr': '172.21.15.181:6823/94712', 'state': ['exists', 'up']}, {'osd': 11, 'uuid': '36e97302-d025-4aaf-ade4-3ece93f0ffd6', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6824', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6825', 'nonce': 94977}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6826', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6827', 'nonce': 94977}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6830', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6831', 'nonce': 94977}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6828', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6829', 'nonce': 94977}]}, 'public_addr': '172.21.15.203:6825/94977', 'cluster_addr': '172.21.15.203:6827/94977', 'heartbeat_back_addr': '172.21.15.203:6831/94977', 'heartbeat_front_addr': '172.21.15.203:6829/94977', 'state': ['exists', 'up']}, {'osd': 12, 'uuid': 'bb2a234d-18d7-4ed3-b250-573eeb31cc0b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6818', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6819', 'nonce': 95092}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6820', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6821', 'nonce': 95092}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6824', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6825', 'nonce': 95092}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6822', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6823', 'nonce': 95092}]}, 'public_addr': '172.21.15.35:6819/95092', 'cluster_addr': '172.21.15.35:6821/95092', 'heartbeat_back_addr': '172.21.15.35:6825/95092', 'heartbeat_front_addr': '172.21.15.35:6823/95092', 'state': ['exists', 'up']}, {'osd': 13, 'uuid': '34d7ae22-e68e-4faa-b647-b251e0b7336b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6816', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6818', 'nonce': 94720}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6820', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6822', 'nonce': 94720}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6828', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6830', 'nonce': 94720}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6824', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6826', 'nonce': 94720}]}, 'public_addr': '172.21.15.83:6818/94720', 'cluster_addr': '172.21.15.83:6822/94720', 'heartbeat_back_addr': '172.21.15.83:6830/94720', 'heartbeat_front_addr': '172.21.15.83:6826/94720', 'state': ['exists', 'up']}, {'osd': 14, 'uuid': '72801aa3-e45b-4a6f-9f52-d327b2caa7aa', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6820', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6822', 'nonce': 94713}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6824', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6826', 'nonce': 94713}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6830', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6831', 'nonce': 94713}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6828', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6829', 'nonce': 94713}]}, 'public_addr': '172.21.15.181:6822/94713', 'cluster_addr': '172.21.15.181:6826/94713', 'heartbeat_back_addr': '172.21.15.181:6831/94713', 'heartbeat_front_addr': '172.21.15.181:6829/94713', 'state': ['exists', 'up']}, {'osd': 15, 'uuid': '99ba406f-ecd6-46eb-bddb-66ae54f63dcf', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6808', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6809', 'nonce': 94976}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6810', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6811', 'nonce': 94976}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6814', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6815', 'nonce': 94976}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6812', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6813', 'nonce': 94976}]}, 'public_addr': '172.21.15.203:6809/94976', 'cluster_addr': '172.21.15.203:6811/94976', 'heartbeat_back_addr': '172.21.15.203:6815/94976', 'heartbeat_front_addr': '172.21.15.203:6813/94976', 'state': ['exists', 'up']}], 'osd_xinfo': [{'osd': 0, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 1, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 2, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 3, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 72}, {'osd': 4, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 5, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 6, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 7, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 8, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 9, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 10, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 11, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 12, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 13, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 14, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 15, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}], 'pg_upmap': [], 'pg_upmap_items': [], 'pg_upmap_primaries': [], 'pg_temp': [], 'primary_temp': [], 'blocklist': {}, 'range_blocklist': {}, 'erasure_code_profiles': {'default': {'crush-failure-domain': 'osd', 'k': '2', 'm': '1', 'plugin': 'jerasure', 'technique': 'reed_sol_van'}, 'shecprofile': {'c': '2', 'crush-device-class': '', 'crush-failure-domain': 'osd', 'crush-root': 'default', 'k': '4', 'm': '3', 'name': 'shecprofile', 'plugin': 'shec', 'technique': 'multiple'}}, 'removed_snaps_queue': [{'pool': 3, 'snaps': [{'begin': 2, 'length': 2}, {'begin': 5, 'length': 2}, {'begin': 10, 'length': 1}, {'begin': 12, 'length': 2}, {'begin': 15, 'length': 4}, {'begin': 23, 'length': 2}, {'begin': 27, 'length': 1}, {'begin': 29, 'length': 1}, {'begin': 31, 'length': 1}, {'begin': 34, 'length': 2}, {'begin': 39, 'length': 1}]}], 'new_removed_snaps': [], 'new_purged_snaps': [], 'crush_node_flags': {}, 'device_class_flags': {}, 'stretch_mode': {'stretch_mode_enabled': False, 'stretch_bucket_count': 0, 'degraded_stretch_mode': 0, 'recovering_stretch_mode': 0, 'stretch_mode_bucket': 0}} Ignoring malparsed backtrace: 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:2f snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {27, 28, 29, 2c, 2d, 2e, 2f} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:26 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {22, 23, 26} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:20 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {1e, 1f, 20} ...repaired Ignoring malparsed backtrace: 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.712749+0000", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.681262+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.1990999999999999e-05 , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.713837+0000", , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.0708e-05 , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.806010+0000", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.814067+0000", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.3025000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.858217+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:58.864 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd set noscrub , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:376: writing smithi20399497-48 from 2457600 to 2621440 tid 1 , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:377: rmattr oid 19 current snap is 33 Ignoring malparsed backtrace: 2023-03-30T00:59:59.089 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 16 v 50 (ObjNum 134 snap 26 seq_num 134) dirty exists , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 1 to smithi20399497-17 , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 2 to smithi20399497-17 Ignoring malparsed backtrace: 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0011711810000000001 , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "commit sent; apply or cleanup", , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "client_info": { Ignoring malparsed backtrace: 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606447+0000", , 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.3420000000000001e-06 , 2023-03-30T00:59:59.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.613875+0000", , 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0042258850000000004 , 2023-03-30T00:59:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.691580+0000", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999971 Ignoring malparsed backtrace: 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.688990+0000", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0050633730000000004 Ignoring malparsed backtrace: 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732738+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.9600000000000006e-06 , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001601318 , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.830761+0000", , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.569096+0000", , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.6s5 36/30 ECSubWrite(tid=17, reqid=client.4555.0:186, at_version=36'9, trim_to=0'0, roll_forward_to=33'6))", Ignoring malparsed backtrace: 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.598900+0000", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.8109999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.603403+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.4739000000000001e-05 , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732868+0000", , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.4030000000000007e-06 , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.034 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:38.760206+0000", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:00.047 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.845424+0000", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.2374999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.854031+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.214 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops , 2023-03-30T01:00:00.274+0000 7f6b68ab7700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 94966) UID: 0 , 2023-03-30T01:00:00.320 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false Ignoring malparsed backtrace: 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:175 3.as0 3:5d617aee:::smithi20399497-20:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e36)", , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:07.550221+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "tid": 388 , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T01:00:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606225+0000", , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.673419+0000", , 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.0645e-05 , 2023-03-30T01:00:00.530 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.703355+0000", , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 7.6669999999999996e-06 , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:705 3.fs0 3:f76751ac:::smithi20399497-42:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e67)", Ignoring malparsed backtrace: 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.800579+0000", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.00046025699999999999 Ignoring malparsed backtrace: 2023-03-30T01:00:00.580 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.805906+0000", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001011241 Ignoring malparsed backtrace: 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 17.667686744000001, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.014015517, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.808867+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "started", , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.811121+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.631 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } Ignoring malparsed backtrace: 2023-03-30T01:00:00.726 INFO:teuthology.orchestra.run.smithi035.stdout:osd.13: {} , 2023-03-30T01:00:00.746 INFO:teuthology.orchestra.run.smithi035.stdout:osd.14: {} , 2023-03-30T01:00:00.767 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T01:00:01.589+0000 7f5cfc24b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 10 (PID: 94706) UID: 0 , 2023-03-30T01:00:01.614 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument Ignoring malparsed backtrace: 2023-03-30T01:00:03.108 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} , 2023-03-30T01:00:03.128 INFO:teuthology.orchestra.run.smithi035.stdout:osd.11: {} , 2023-03-30T01:00:03.133 INFO:teuthology.orchestra.run.smithi203.stdout:{ Ignoring malparsed backtrace: 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.332416+0000", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1637999999999999e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.812171+0000", , 2023-03-30T01:00:03.480 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.3927e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.500 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.820021+0000", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 2.446e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T01:00:03.515 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 140 Ignoring malparsed backtrace: 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144542+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.542 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.381561+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384196+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_info": { , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", Ignoring malparsed backtrace: 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:03.611 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", Ignoring malparsed backtrace: 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.5550000000000002e-06 , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000104752 , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.705642+0000", , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.759581+0000", , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1364e-05 , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0038820090000000001 , 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.138 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", , 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:04.153 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.628576+0000", , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 7.4080000000000001e-06 , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.191 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.635982+0000", , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0049630509999999996 , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.721032+0000", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.724382+0000", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 9.4830000000000005e-06 Ignoring malparsed backtrace: 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.731143+0000", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001987801 Ignoring malparsed backtrace: 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.726052+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:33.738720+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.767733+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T01:00:04.342 INFO:teuthology.orchestra.run.smithi203.stdout: "description": "osd_op(client.4555.0:780 3.13s0 3:cbc5c355:::smithi20399497-3:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e72)", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "initiated_at": "2023-03-30T00:59:43.841834+0000", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "age": 20.282786238, Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x135) [0x55a59601f2ad] , 2: ceph-osd(+0x631473) [0x55a59601f473] Ignoring malparsed backtrace: 2023-03-30T00:59:43.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:58:48.149595+0000", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op must wait for map", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.575214+0000", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7829999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 26.474046659999999, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062746649999999996, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:16.606341+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_started", , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:23.708494+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:23.712400+0000", , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.367927426000001, , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076374399999999997, Ignoring malparsed backtrace: 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010535 , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:541 3.1es0 3:7ef085b3:::smithi20399497-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:26.715419+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703128+0000", , 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2414e-05 , 2023-03-30T00:59:43.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703333+0000", , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012205999999999999 , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.774152+0000", , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00084599299999999996 , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, , 2023-03-30T00:59:43.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, Ignoring malparsed backtrace: 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0399999999999998e-06 , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.585362+0000", , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.614174+0000", , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7903e-05 , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.678976+0000", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.683130+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.690158+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpWrite(3.as5 71/54 ECSubWrite(tid=203, reqid=client.4555.0:735, at_version=71'38, trim_to=0'0, roll_forward_to=71'35))", , 2023-03-30T00:59:43.801 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.774378+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.807553+0000", , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012399939999999999 , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.812386+0000", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 Ignoring malparsed backtrace: 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 43 v 48 (ObjNum 129 snap 24 seq_num 129) dirty exists , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:300: left oid 43 (ObjNum 129 snap 24 seq_num 129) , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:302: finishing write tid 1 to smithi20399497-3 Ignoring malparsed backtrace: 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.581882+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.587173+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068317159999999998 , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:18.626041+0000", , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.059e-05 , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.685149+0000", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.758157+0000", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068448800000000002 Ignoring malparsed backtrace: 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.809042+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.440 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", Ignoring malparsed backtrace: 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:44.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00063433800000000002 , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.617 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768302+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.773+0000 7f7e20ae9700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:756d2f96:::smithi20399497-24:c repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:c63dffb4:::smithi20399497-40:6 repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{12=[12,10,d]} for object 3:cb5d7287:::smithi20399497-32:c Ignoring malparsed backtrace: 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.600924+0000", , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000208836 , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:44.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665069+0000", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665141+0000", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2509999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:44.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703115+0000", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0409000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.743640+0000", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076826619999999998 Ignoring malparsed backtrace: 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.755311+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.766273+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.4555", , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", Ignoring malparsed backtrace: 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", Ignoring malparsed backtrace: 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9876999999999993e-05 , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.792080+0000", , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8390000000000003e-06 , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:744 3.10s0 3:0aab8796:::smithi20399497-28:head [read 0~0,checksum] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e71)", Ignoring malparsed backtrace: 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.814167+0000", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2109999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:45.731+0000 7efdb31a1700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 11 (PID: 94971) UID: 0 , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:307: done (2 left) , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:308: rollback oid 45 current snap is 25 Ignoring malparsed backtrace: 2023-03-30T00:59:46.791+0000 7f2881c66700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:ef8379c0:::smithi20399497-18:5 repair needed (no obc) , 2023-03-30T00:59:46.791+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{11=[11,10,f,d]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:46.789+0000 7f579d0a4700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:5d617aee:::smithi20399497-20:3 repair needed (no obc) Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T00:59:47.581 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true , 2023-03-30T00:59:47.650+0000 7f0f9634b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 95055) UID: 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.5651e-05 , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0048024890000000001 , 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.888 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.900 INFO:teuthology.orchestra.run.smithi035.stdout: "tid": 249 , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.605832+0000", , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0661e-05 , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.926 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.618433+0000", , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.2633999999999995e-05 , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707573+0000", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707643+0000", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.7330000000000003e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "reached_pg", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680627+0000", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010247900000000001 Ignoring malparsed backtrace: 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.690506+0000", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.9204999999999997e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705513+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "op must wait for map", , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705691+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.710594+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.717126+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:48.075 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} , 2023-03-30T00:59:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 , 2023-03-30T00:59:48.076 INFO:tasks.rados.rados.0.smithi203.stdout:324: writing smithi20399497-19 from 2293760 to 3047424 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "complaint_time": 30, , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "num_blocked_ops": 0 , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010960500000000001 , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:527 3.14s0 3:2e5a366f:::smithi20399497-31:head [append 0~0] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:26.695461+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.698204+0000", , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.704831+0000", , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 9.7558000000000003e-05 , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:534 3.4s0 3:24eec11e:::smithi20399497-7:head [setxattr _header (56) in=7b] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", Ignoring malparsed backtrace: 2023-03-30T00:59:48.646 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.705518+0000", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.691686+0000", , 2023-03-30T00:59:48.671 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.01079631 Ignoring malparsed backtrace: 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680770+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.698 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "started", , 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "client_info": { , 2023-03-30T00:59:48.712 INFO:teuthology.orchestra.run.smithi035.stdout: "client": "client.4555", Ignoring malparsed backtrace: 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.2860000000000002e-06 , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0061165480000000003 , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: } , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:48.833+0000 7efd99558700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:14c33e9f:::smithi20399497-34:3 repair needed (no obc) , 2023-03-30T00:59:48.836 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 0 to 802816 tid 1 , 2023-03-30T00:59:48.838 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 802816 to 1228800 tid 2 Ignoring malparsed backtrace: 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ , 2023-03-30T00:59:49.307 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0722e-05 , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.604113+0000", , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00052637799999999998 , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0021209829999999999, , 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "type_data": { , 2023-03-30T00:59:49.363 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.610251+0000", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.7749999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "all_read", , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.608123+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:17.611077+0000", , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "age": 31.691323830999998, , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0017984959999999999, Ignoring malparsed backtrace: 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.789051+0000", , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.796145+0000", , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0040419669999999996 , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.491 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.798914+0000", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.9406e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 523, in _write_all , count = self._write(data) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1364, in _write Ignoring malparsed backtrace: 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "initiated", , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.560616+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.583527+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.2229999999999996e-06 , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.683455+0000", , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.7909999999999999e-06 , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.680363+0000", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.687466+0000", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.7453e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:50.135 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.765837+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.1ds1 77/54 ECSubWrite(tid=239, reqid=client.4555.0:880, at_version=77'66, trim_to=0'0, roll_forward_to=77'64))", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:48.854994+0000", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 1.1714130300000001, Ignoring malparsed backtrace: 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.8422000000000001e-05 , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.486+0000 7f6a8da31700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 14 (PID: 94707) UID: 0 , 2023-03-30T00:59:50.492 INFO:teuthology.orchestra.run.smithi035.stdout:osd.9: {} , 2023-03-30T00:59:50.513 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} Ignoring malparsed backtrace: 2023-03-30T00:59:51.270 INFO:teuthology.orchestra.run.smithi083.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:51.271 INFO:teuthology.orchestra.run.smithi083.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. , 2023-03-30T00:59:51.280 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops Ignoring malparsed backtrace: 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 1 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 2 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:348: finishing copy_from to smithi20399497-15 Ignoring malparsed backtrace: 2023-03-30T00:59:53.011+0000 7f57b74ee700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 13 (PID: 94715) UID: 0 , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. Ignoring malparsed backtrace: 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.599544+0000", , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0852000000000005e-05 , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.684493+0000", , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121767 , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.040131732000000003, , 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { , 2023-03-30T00:59:53.381 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2023-03-30T00:59:53.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703140+0000", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.756116+0000", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105009 Ignoring malparsed backtrace: 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:53.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779826+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779943+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: ] , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0249999999999998e-06 , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpReadReply(3.9s0 71/42 ECSubReadReply(tid=160, attrs_read=0))", , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.803584+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768289+0000", , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0489999999999999e-06 , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.521 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 756 , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:48.845453+0000", , 2023-03-30T00:59:53.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.843+0000 7fc5039f5700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 8 (PID: 95060) UID: 0 , 2023-03-30T00:59:53.847 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 13 1 , 2023-03-30T00:59:53.884+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0117e-05 , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.654122+0000", , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.688971+0000", , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0037431610000000001 , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.015 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.710655+0000", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.6818000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.743870+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.041 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.754628+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0947e-05 , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.813138+0000", , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.9743999999999999e-05 , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0033700029999999999, , 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:54.106 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_started", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.824625+0000", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.8889999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000211175 , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 3.8890000000000002e-05 , 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.630 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 37 , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:54.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144504+0000", , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.0872999999999996e-05 , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.149407+0000", , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.8856000000000001e-05 , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384053+0000", , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.2444999999999998e-05 , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 187 , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.374736+0000", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.4740000000000002e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384434+0000", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.2589000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.581520+0000", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000112135 Ignoring malparsed backtrace: 2023-03-30T00:59:54.777 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578853+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578887+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.630036+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.833+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 20=[]:{12=[10]} for object 3:6d0110e8:::smithi20399497-36:c , 2023-03-30T00:59:54.834+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) , 2023-03-30T00:59:54.862 DEBUG:teuthology.orchestra.run.smithi203:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.15.asok dump_ops_in_flight Ignoring malparsed backtrace: 2023-03-30T00:59:55.276 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.622353+0000", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001925162 Ignoring malparsed backtrace: 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.302 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.629137+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.636040+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:55.353 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.6981000000000001e-05 , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.00041520000000000001 , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.809116+0000", , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.8238999999999999e-05 , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.773072+0000", , 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000102753 , 2023-03-30T00:59:55.418 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.786664+0000", , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0077388860000000004 , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0076006499999999996, , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "started", Ignoring malparsed backtrace: 2023-03-30T00:59:55.455 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.827928+0000", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.1890000000000007e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:56.235 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops , 2023-03-30T00:59:56.266 ERROR:teuthology.orchestra.daemon.state:Failed to send signal 1: None , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-30T00:59:56.868+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{11=[11,10,f,d],2c=[2c,29,28,27,26,23,20,1e,1c]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:56.870+0000 7fca99663700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{d=[d],20=[20,1f,1e,1d,1c,1b,17],2c=[2c,29,28,27,26,23]} for object 3:ac746cfe:::smithi20399497-5:c , 2023-03-30T00:59:56.870+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:bbe97148:::smithi20399497-47:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:57.884 INFO:teuthology.orchestra.run.smithi035.stdout:osd.0: {} , 2023-03-30T00:59:57.885+0000 7f7e40f3f700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 2 (PID: 94700) UID: 0 , 2023-03-30T00:59:57.888 INFO:tasks.rados.rados.0.smithi203.stdout:368: writing smithi20399497-3 from 4521984 to 4636672 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_change": "24", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend": "0", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend_prenautilus": "0", Ignoring malparsed backtrace: 2023-03-30T00:59:58.075 INFO:teuthology.orchestra.run.smithi035.stdout: "write_tier": -1, , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "cache_mode": "none", , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "target_max_bytes": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_grade_decay_rate": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_search_last_n": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "grade_table": [], Ignoring malparsed backtrace: 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "up_thru": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "down_at": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "lost_at": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.130 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", Ignoring malparsed backtrace: 2023-03-30T00:59:58.146 INFO:teuthology.orchestra.run.smithi035.stdout: "addrvec": [ , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", Ignoring malparsed backtrace: 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "uuid": "32fd7d07-dd9a-476c-9fc4-da8e1f2855c0", , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "up": 1, , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "in": 1, Ignoring malparsed backtrace: 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: "cluster_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6807", Ignoring malparsed backtrace: 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addr": "172.21.15.35:6809/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_front_addr": "172.21.15.35:6807/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "state": [ Ignoring malparsed backtrace: 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.181:6816", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94712 Ignoring malparsed backtrace: 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.243 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.35:6823", Ignoring malparsed backtrace: 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_begin": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_end": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "up_from": 22, Ignoring malparsed backtrace: 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6810", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94976 Ignoring malparsed backtrace: 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "osd": 5, , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_interval": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout: "num_ops": 0 , 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout:} , 2023-03-30T00:59:58.312 INFO:tasks.thrashosds.thrasher:j is {'epoch': 86, 'fsid': '8f2049d8-2bb0-46fe-a814-ce26c43a7f40', 'created': '2023-03-30T00:58:33.472724+0000', 'modified': '2023-03-30T00:59:57.844713+0000', 'last_up_change': '2023-03-30T00:59:43.755794+0000', 'last_in_change': '2023-03-30T00:58:39.154518+0000', 'flags': 'sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit', 'flags_num': 5799936, 'flags_set': ['pglog_hardlimit', 'purged_snapdirs', 'recovery_deletes', 'sortbitwise'], 'crush_version': 7, 'full_ratio': 0.949999988079071, 'backfillfull_ratio': 0.8999999761581421, 'nearfull_ratio': 0.8500000238418579, 'cluster_snapshot': '', 'pool_max': 3, 'max_osd': 16, 'require_min_compat_client': 'luminous', 'min_compat_client': 'jewel', 'require_osd_release': 'reef', 'allow_crimson': False, 'pools': [{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-30T00:58:45.356303+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '24', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 15.380000114440918, 'score_stable': 15.380000114440918, 'optimal_score': 0.12999999523162842, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 2, 'pool_name': 'rbd', 'create_time': '2023-03-30T00:58:49.826143+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '27', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}, 'read_balance': {'score_acting': 3.7699999809265137, 'score_stable': 3.7699999809265137, 'optimal_score': 0.7300000190734863, 'raw_score_acting': 2.75, 'raw_score_stable': 2.75, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 3, 'pool_name': 'unique_pool_0', 'create_time': '2023-03-30T00:59:00.541501+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 3, 'size': 7, 'min_size': 5, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 36, 'pg_placement_num': 16, 'pg_placement_num_target': 36, 'pg_num_target': 36, 'pg_num_pending': 36, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '54', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 54, 'snap_epoch': 86, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'shecprofile', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 16384, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rados': {}}}], 'osds': [{'osd': 0, 'uuid': 'ebbb26ab-20a8-4846-8b3f-e2fbabb7cf75', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 42, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6810', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6811', 'nonce': 95086}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6812', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6813', 'nonce': 95086}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6816', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6817', 'nonce': 95086}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6814', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6815', 'nonce': 95086}]}, 'public_addr': '172.21.15.35:6811/95086', 'cluster_addr': '172.21.15.35:6813/95086', 'heartbeat_back_addr': '172.21.15.35:6817/95086', 'heartbeat_front_addr': '172.21.15.35:6815/95086', 'state': ['exists', 'up']}, {'osd': 1, 'uuid': '3bc34814-68b2-4d01-8249-953700168560', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6800', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6801', 'nonce': 94718}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6802', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6803', 'nonce': 94718}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6806', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6807', 'nonce': 94718}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6804', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6805', 'nonce': 94718}]}, 'public_addr': '172.21.15.83:6801/94718', 'cluster_addr': '172.21.15.83:6803/94718', 'heartbeat_back_addr': '172.21.15.83:6807/94718', 'heartbeat_front_addr': '172.21.15.83:6805/94718', 'state': ['exists', 'up']}, {'osd': 2, 'uuid': '76f71867-1d44-4527-877a-fd92aac57333', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6808', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6809', 'nonce': 94710}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6810', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6811', 'nonce': 94710}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6814', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6815', 'nonce': 94710}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6812', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6813', 'nonce': 94710}]}, 'public_addr': '172.21.15.181:6809/94710', 'cluster_addr': '172.21.15.181:6811/94710', 'heartbeat_back_addr': '172.21.15.181:6815/94710', 'heartbeat_front_addr': '172.21.15.181:6813/94710', 'state': ['exists', 'up']}, {'osd': 3, 'uuid': '954b6f1d-986f-49e6-93e5-d3423f3bf421', 'up': 0, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 22, 'last_clean_end': 27, 'up_from': 70, 'up_thru': 0, 'down_at': 72, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6816', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6817', 'nonce': 100463}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6818', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6819', 'nonce': 100463}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6822', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6823', 'nonce': 100463}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6820', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6821', 'nonce': 100463}]}, 'public_addr': '172.21.15.203:6817/100463', 'cluster_addr': '172.21.15.203:6819/100463', 'heartbeat_back_addr': '172.21.15.203:6823/100463', 'heartbeat_front_addr': '172.21.15.203:6821/100463', 'state': ['exists']}, {'osd': 4, 'uuid': 'ad653e95-e58a-4fec-9db8-d313042c2e97', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6826', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6827', 'nonce': 95091}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6828', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6829', 'nonce': 95091}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6832', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6833', 'nonce': 95091}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6830', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6831', 'nonce': 95091}]}, 'public_addr': '172.21.15.35:6827/95091', 'cluster_addr': '172.21.15.35:6829/95091', 'heartbeat_back_addr': '172.21.15.35:6833/95091', 'heartbeat_front_addr': '172.21.15.35:6831/95091', 'state': ['exists', 'up']}, {'osd': 5, 'uuid': '32fd7d07-dd9a-476c-9fc4-da8e1f2855c0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6808', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6809', 'nonce': 94717}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6810', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6811', 'nonce': 94717}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6814', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6815', 'nonce': 94717}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6812', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6813', 'nonce': 94717}]}, 'public_addr': '172.21.15.83:6809/94717', 'cluster_addr': '172.21.15.83:6811/94717', 'heartbeat_back_addr': '172.21.15.83:6815/94717', 'heartbeat_front_addr': '172.21.15.83:6813/94717', 'state': ['exists', 'up']}, {'osd': 6, 'uuid': '28a11dac-0c35-4a42-8250-cfedff301ba0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 0, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6800', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6801', 'nonce': 94711}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6802', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6803', 'nonce': 94711}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6806', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6807', 'nonce': 94711}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6804', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6805', 'nonce': 94711}]}, 'public_addr': '172.21.15.181:6801/94711', 'cluster_addr': '172.21.15.181:6803/94711', 'heartbeat_back_addr': '172.21.15.181:6807/94711', 'heartbeat_front_addr': '172.21.15.181:6805/94711', 'state': ['exists', 'up']}, {'osd': 7, 'uuid': 'ea60440a-7f79-4503-9ddb-b513c366c68a', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6800', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6801', 'nonce': 94975}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6802', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6803', 'nonce': 94975}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6806', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6807', 'nonce': 94975}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6804', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6805', 'nonce': 94975}]}, 'public_addr': '172.21.15.203:6801/94975', 'cluster_addr': '172.21.15.203:6803/94975', 'heartbeat_back_addr': '172.21.15.203:6807/94975', 'heartbeat_front_addr': '172.21.15.203:6805/94975', 'state': ['exists', 'up']}, {'osd': 8, 'uuid': '2614c844-ce70-4875-86cd-7ce40b5002db', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6802', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6803', 'nonce': 95087}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6804', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6805', 'nonce': 95087}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6808', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6809', 'nonce': 95087}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6806', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6807', 'nonce': 95087}]}, 'public_addr': '172.21.15.35:6803/95087', 'cluster_addr': '172.21.15.35:6805/95087', 'heartbeat_back_addr': '172.21.15.35:6809/95087', 'heartbeat_front_addr': '172.21.15.35:6807/95087', 'state': ['exists', 'up']}, {'osd': 9, 'uuid': '110531cf-e218-4332-aef0-9f05d71178bb', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6817', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6819', 'nonce': 94719}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6821', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6823', 'nonce': 94719}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6829', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6831', 'nonce': 94719}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6825', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6827', 'nonce': 94719}]}, 'public_addr': '172.21.15.83:6819/94719', 'cluster_addr': '172.21.15.83:6823/94719', 'heartbeat_back_addr': '172.21.15.83:6831/94719', 'heartbeat_front_addr': '172.21.15.83:6827/94719', 'state': ['exists', 'up']}, {'osd': 10, 'uuid': 'c7b567e6-1561-4f29-bbd4-65a6a4746929', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6816', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6817', 'nonce': 94712}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6818', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6819', 'nonce': 94712}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6825', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6827', 'nonce': 94712}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6821', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6823', 'nonce': 94712}]}, 'public_addr': '172.21.15.181:6817/94712', 'cluster_addr': '172.21.15.181:6819/94712', 'heartbeat_back_addr': '172.21.15.181:6827/94712', 'heartbeat_front_addr': '172.21.15.181:6823/94712', 'state': ['exists', 'up']}, {'osd': 11, 'uuid': '36e97302-d025-4aaf-ade4-3ece93f0ffd6', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6824', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6825', 'nonce': 94977}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6826', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6827', 'nonce': 94977}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6830', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6831', 'nonce': 94977}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6828', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6829', 'nonce': 94977}]}, 'public_addr': '172.21.15.203:6825/94977', 'cluster_addr': '172.21.15.203:6827/94977', 'heartbeat_back_addr': '172.21.15.203:6831/94977', 'heartbeat_front_addr': '172.21.15.203:6829/94977', 'state': ['exists', 'up']}, {'osd': 12, 'uuid': 'bb2a234d-18d7-4ed3-b250-573eeb31cc0b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6818', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6819', 'nonce': 95092}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6820', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6821', 'nonce': 95092}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6824', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6825', 'nonce': 95092}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6822', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6823', 'nonce': 95092}]}, 'public_addr': '172.21.15.35:6819/95092', 'cluster_addr': '172.21.15.35:6821/95092', 'heartbeat_back_addr': '172.21.15.35:6825/95092', 'heartbeat_front_addr': '172.21.15.35:6823/95092', 'state': ['exists', 'up']}, {'osd': 13, 'uuid': '34d7ae22-e68e-4faa-b647-b251e0b7336b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6816', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6818', 'nonce': 94720}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6820', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6822', 'nonce': 94720}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6828', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6830', 'nonce': 94720}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6824', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6826', 'nonce': 94720}]}, 'public_addr': '172.21.15.83:6818/94720', 'cluster_addr': '172.21.15.83:6822/94720', 'heartbeat_back_addr': '172.21.15.83:6830/94720', 'heartbeat_front_addr': '172.21.15.83:6826/94720', 'state': ['exists', 'up']}, {'osd': 14, 'uuid': '72801aa3-e45b-4a6f-9f52-d327b2caa7aa', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6820', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6822', 'nonce': 94713}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6824', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6826', 'nonce': 94713}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6830', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6831', 'nonce': 94713}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6828', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6829', 'nonce': 94713}]}, 'public_addr': '172.21.15.181:6822/94713', 'cluster_addr': '172.21.15.181:6826/94713', 'heartbeat_back_addr': '172.21.15.181:6831/94713', 'heartbeat_front_addr': '172.21.15.181:6829/94713', 'state': ['exists', 'up']}, {'osd': 15, 'uuid': '99ba406f-ecd6-46eb-bddb-66ae54f63dcf', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6808', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6809', 'nonce': 94976}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6810', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6811', 'nonce': 94976}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6814', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6815', 'nonce': 94976}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6812', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6813', 'nonce': 94976}]}, 'public_addr': '172.21.15.203:6809/94976', 'cluster_addr': '172.21.15.203:6811/94976', 'heartbeat_back_addr': '172.21.15.203:6815/94976', 'heartbeat_front_addr': '172.21.15.203:6813/94976', 'state': ['exists', 'up']}], 'osd_xinfo': [{'osd': 0, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 1, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 2, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 3, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 72}, {'osd': 4, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 5, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 6, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 7, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 8, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 9, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 10, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 11, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 12, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 13, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 14, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 15, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}], 'pg_upmap': [], 'pg_upmap_items': [], 'pg_upmap_primaries': [], 'pg_temp': [], 'primary_temp': [], 'blocklist': {}, 'range_blocklist': {}, 'erasure_code_profiles': {'default': {'crush-failure-domain': 'osd', 'k': '2', 'm': '1', 'plugin': 'jerasure', 'technique': 'reed_sol_van'}, 'shecprofile': {'c': '2', 'crush-device-class': '', 'crush-failure-domain': 'osd', 'crush-root': 'default', 'k': '4', 'm': '3', 'name': 'shecprofile', 'plugin': 'shec', 'technique': 'multiple'}}, 'removed_snaps_queue': [{'pool': 3, 'snaps': [{'begin': 2, 'length': 2}, {'begin': 5, 'length': 2}, {'begin': 10, 'length': 1}, {'begin': 12, 'length': 2}, {'begin': 15, 'length': 4}, {'begin': 23, 'length': 2}, {'begin': 27, 'length': 1}, {'begin': 29, 'length': 1}, {'begin': 31, 'length': 1}, {'begin': 34, 'length': 2}, {'begin': 39, 'length': 1}]}], 'new_removed_snaps': [], 'new_purged_snaps': [], 'crush_node_flags': {}, 'device_class_flags': {}, 'stretch_mode': {'stretch_mode_enabled': False, 'stretch_bucket_count': 0, 'degraded_stretch_mode': 0, 'recovering_stretch_mode': 0, 'stretch_mode_bucket': 0}} Ignoring malparsed backtrace: 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:2f snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {27, 28, 29, 2c, 2d, 2e, 2f} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:26 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {22, 23, 26} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:20 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {1e, 1f, 20} ...repaired Ignoring malparsed backtrace: 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.712749+0000", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.681262+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.1990999999999999e-05 , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.713837+0000", , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.0708e-05 , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.806010+0000", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.814067+0000", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.3025000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.858217+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:58.864 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd set noscrub , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:376: writing smithi20399497-48 from 2457600 to 2621440 tid 1 , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:377: rmattr oid 19 current snap is 33 Ignoring malparsed backtrace: 2023-03-30T00:59:59.089 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 16 v 50 (ObjNum 134 snap 26 seq_num 134) dirty exists , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 1 to smithi20399497-17 , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 2 to smithi20399497-17 Ignoring malparsed backtrace: 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0011711810000000001 , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "commit sent; apply or cleanup", , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "client_info": { Ignoring malparsed backtrace: 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606447+0000", , 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.3420000000000001e-06 , 2023-03-30T00:59:59.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.613875+0000", , 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0042258850000000004 , 2023-03-30T00:59:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.691580+0000", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999971 Ignoring malparsed backtrace: 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.688990+0000", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0050633730000000004 Ignoring malparsed backtrace: 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732738+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.9600000000000006e-06 , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001601318 , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.830761+0000", , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.569096+0000", , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.6s5 36/30 ECSubWrite(tid=17, reqid=client.4555.0:186, at_version=36'9, trim_to=0'0, roll_forward_to=33'6))", Ignoring malparsed backtrace: 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.598900+0000", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.8109999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.603403+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.4739000000000001e-05 , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732868+0000", , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.4030000000000007e-06 , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.034 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:38.760206+0000", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:00.047 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.845424+0000", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.2374999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.854031+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.214 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops , 2023-03-30T01:00:00.274+0000 7f6b68ab7700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 94966) UID: 0 , 2023-03-30T01:00:00.320 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false Ignoring malparsed backtrace: 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:175 3.as0 3:5d617aee:::smithi20399497-20:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e36)", , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:07.550221+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "tid": 388 , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T01:00:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606225+0000", , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.673419+0000", , 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.0645e-05 , 2023-03-30T01:00:00.530 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.703355+0000", , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 7.6669999999999996e-06 , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:705 3.fs0 3:f76751ac:::smithi20399497-42:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e67)", Ignoring malparsed backtrace: 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.800579+0000", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.00046025699999999999 Ignoring malparsed backtrace: 2023-03-30T01:00:00.580 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.805906+0000", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001011241 Ignoring malparsed backtrace: 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 17.667686744000001, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.014015517, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.808867+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "started", , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.811121+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.631 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } Ignoring malparsed backtrace: 2023-03-30T01:00:00.726 INFO:teuthology.orchestra.run.smithi035.stdout:osd.13: {} , 2023-03-30T01:00:00.746 INFO:teuthology.orchestra.run.smithi035.stdout:osd.14: {} , 2023-03-30T01:00:00.767 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T01:00:01.589+0000 7f5cfc24b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 10 (PID: 94706) UID: 0 , 2023-03-30T01:00:01.614 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument Ignoring malparsed backtrace: 2023-03-30T01:00:03.108 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} , 2023-03-30T01:00:03.128 INFO:teuthology.orchestra.run.smithi035.stdout:osd.11: {} , 2023-03-30T01:00:03.133 INFO:teuthology.orchestra.run.smithi203.stdout:{ Ignoring malparsed backtrace: 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.332416+0000", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1637999999999999e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.812171+0000", , 2023-03-30T01:00:03.480 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.3927e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.500 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.820021+0000", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 2.446e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T01:00:03.515 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 140 Ignoring malparsed backtrace: 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144542+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.542 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.381561+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384196+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_info": { , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", Ignoring malparsed backtrace: 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:03.611 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", Ignoring malparsed backtrace: 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.5550000000000002e-06 , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000104752 , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.705642+0000", , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.759581+0000", , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1364e-05 , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0038820090000000001 , 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.138 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", , 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:04.153 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.628576+0000", , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 7.4080000000000001e-06 , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.191 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.635982+0000", , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0049630509999999996 , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.721032+0000", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.724382+0000", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 9.4830000000000005e-06 Ignoring malparsed backtrace: 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.731143+0000", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001987801 Ignoring malparsed backtrace: 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.726052+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:33.738720+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.767733+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T01:00:04.342 INFO:teuthology.orchestra.run.smithi203.stdout: "description": "osd_op(client.4555.0:780 3.13s0 3:cbc5c355:::smithi20399497-3:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e72)", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "initiated_at": "2023-03-30T00:59:43.841834+0000", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "age": 20.282786238, Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x135) [0x55a59601f2ad] , 2: ceph-osd(+0x631473) [0x55a59601f473] Ignoring malparsed backtrace: 2023-03-30T00:59:43.096 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:58:48.149595+0000", , 2023-03-30T00:59:43.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "op must wait for map", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.575214+0000", , 2023-03-30T00:59:43.115 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.7829999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 26.474046659999999, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0062746649999999996, , 2023-03-30T00:59:43.128 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:43.141 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:16.606341+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_started", , 2023-03-30T00:59:43.154 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:23.708494+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:23.712400+0000", , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "age": 19.367927426000001, , 2023-03-30T00:59:43.167 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076374399999999997, Ignoring malparsed backtrace: 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.180 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.192 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.193 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.205 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00010535 , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.218 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:541 3.1es0 3:7ef085b3:::smithi20399497-2:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:43.231 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:26.715419+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703128+0000", , 2023-03-30T00:59:43.243 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.2414e-05 , 2023-03-30T00:59:43.244 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703333+0000", , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00012205999999999999 , 2023-03-30T00:59:43.256 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.774152+0000", , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00084599299999999996 , 2023-03-30T00:59:43.269 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:43.683 INFO:teuthology.orchestra.run.smithi181.stdout: "size": 20, , 2023-03-30T00:59:43.684 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 600, Ignoring malparsed backtrace: 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 9.0399999999999998e-06 , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.697 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.585362+0000", , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:43.710 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:12.614174+0000", , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.7903e-05 , 2023-03-30T00:59:43.723 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.678976+0000", , 2023-03-30T00:59:43.736 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.6410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:43.749 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.683130+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:43.762 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.690158+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.775 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:43.788 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:43.800 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpWrite(3.as5 71/54 ECSubWrite(tid=203, reqid=client.4555.0:735, at_version=71'38, trim_to=0'0, roll_forward_to=71'35))", , 2023-03-30T00:59:43.801 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.774378+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.807553+0000", , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0012399939999999999 , 2023-03-30T00:59:43.813 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "header_read", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.812386+0000", , 2023-03-30T00:59:43.827 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 4294967295.999999 Ignoring malparsed backtrace: 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:43.840 INFO:teuthology.orchestra.run.smithi181.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 43 v 48 (ObjNum 129 snap 24 seq_num 129) dirty exists , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:300: left oid 43 (ObjNum 129 snap 24 seq_num 129) , 2023-03-30T00:59:43.865 INFO:tasks.rados.rados.0.smithi203.stdout:302: finishing write tid 1 to smithi20399497-3 Ignoring malparsed backtrace: 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.296 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", , 2023-03-30T00:59:44.297 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.581882+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.310 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "done", , 2023-03-30T00:59:44.311 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:07.587173+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.330 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.343 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068317159999999998 , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.358 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:18.626041+0000", , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 1.059e-05 , 2023-03-30T00:59:44.375 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.685149+0000", , 2023-03-30T00:59:44.390 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.758157+0000", , 2023-03-30T00:59:44.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0068448800000000002 Ignoring malparsed backtrace: 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "dispatched", , 2023-03-30T00:59:44.427 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.809042+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.440 INFO:teuthology.orchestra.run.smithi181.stdout: "events": [ , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.441 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", Ignoring malparsed backtrace: 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.454 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:44.469 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.00063433800000000002 , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:44.470 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.617 INFO:teuthology.orchestra.run.smithi181.stdout:{ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: "ops": [ , 2023-03-30T00:59:44.618 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.631 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768302+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:44.773+0000 7f7e20ae9700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:756d2f96:::smithi20399497-24:c repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:c63dffb4:::smithi20399497-40:6 repair needed (no obc) , 2023-03-30T00:59:44.773+0000 7f8a80222700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{12=[12,10,d]} for object 3:cb5d7287:::smithi20399497-32:c Ignoring malparsed backtrace: 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.600924+0000", , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000208836 , 2023-03-30T00:59:44.928 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:44.944 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665069+0000", , 2023-03-30T00:59:44.945 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.665141+0000", , 2023-03-30T00:59:44.960 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.2509999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:44.977 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703115+0000", , 2023-03-30T00:59:44.978 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 2.0409000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "reached_pg", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.743640+0000", , 2023-03-30T00:59:44.995 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.0076826619999999998 Ignoring malparsed backtrace: 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:45.015 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.755311+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.032 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.766273+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_info": { , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client": "client.4555", , 2023-03-30T00:59:45.045 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", Ignoring malparsed backtrace: 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.058 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", Ignoring malparsed backtrace: 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0 , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.071 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.9876999999999993e-05 , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.084 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.792080+0000", , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8390000000000003e-06 , 2023-03-30T00:59:45.097 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:45.110 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "osd_op(client.4555.0:744 3.10s0 3:0aab8796:::smithi20399497-28:head [read 0~0,checksum] snapc 0=[] RETRY=1 ondisk+retry+read+known_if_redirected+supports_pool_eio e71)", Ignoring malparsed backtrace: 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.814167+0000", , 2023-03-30T00:59:45.123 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 8.2109999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:45.731+0000 7efdb31a1700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 11 (PID: 94971) UID: 0 , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:307: done (2 left) , 2023-03-30T00:59:45.778 INFO:tasks.rados.rados.0.smithi203.stdout:308: rollback oid 45 current snap is 25 Ignoring malparsed backtrace: 2023-03-30T00:59:46.791+0000 7f2881c66700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:ef8379c0:::smithi20399497-18:5 repair needed (no obc) , 2023-03-30T00:59:46.791+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 12=[]:{11=[11,10,f,d]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:46.789+0000 7f579d0a4700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:5d617aee:::smithi20399497-20:3 repair needed (no obc) Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T00:59:47.581 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true , 2023-03-30T00:59:47.650+0000 7f0f9634b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 0 (PID: 95055) UID: 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.5651e-05 , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.873 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0048024890000000001 , 2023-03-30T00:59:47.887 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.888 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:47.900 INFO:teuthology.orchestra.run.smithi035.stdout: "tid": 249 , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:47.901 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.605832+0000", , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0661e-05 , 2023-03-30T00:59:47.914 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.926 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.618433+0000", , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.2633999999999995e-05 , 2023-03-30T00:59:47.927 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707573+0000", , 2023-03-30T00:59:47.940 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:23.707643+0000", , 2023-03-30T00:59:47.953 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.7330000000000003e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "reached_pg", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680627+0000", , 2023-03-30T00:59:47.966 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010247900000000001 Ignoring malparsed backtrace: 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.690506+0000", , 2023-03-30T00:59:47.979 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.9204999999999997e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "throttled", , 2023-03-30T00:59:47.992 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705513+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "op must wait for map", , 2023-03-30T00:59:48.005 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.705691+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:48.018 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.710594+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.031 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:32.717126+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.045 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.058 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_committed", Ignoring malparsed backtrace: 2023-03-30T00:59:48.075 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} , 2023-03-30T00:59:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 6 , 2023-03-30T00:59:48.076 INFO:tasks.rados.rados.0.smithi203.stdout:324: writing smithi20399497-19 from 2293760 to 3047424 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "complaint_time": 30, , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout: "num_blocked_ops": 0 , 2023-03-30T00:59:48.387 INFO:teuthology.orchestra.run.smithi035.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.546 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00010960500000000001 , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.561 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:527 3.14s0 3:2e5a366f:::smithi20399497-31:head [append 0~0] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", , 2023-03-30T00:59:48.581 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:26.695461+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.698204+0000", , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:48.595 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.704831+0000", , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 9.7558000000000003e-05 , 2023-03-30T00:59:48.612 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.630 INFO:teuthology.orchestra.run.smithi035.stdout: "description": "osd_op(client.4555.0:534 3.4s0 3:24eec11e:::smithi20399497-7:head [setxattr _header (56) in=7b] snapc 17=[17,10] ondisk+write+known_if_redirected+supports_pool_eio e55)", Ignoring malparsed backtrace: 2023-03-30T00:59:48.646 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:26.705518+0000", , 2023-03-30T00:59:48.647 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "done", , 2023-03-30T00:59:48.670 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.691686+0000", , 2023-03-30T00:59:48.671 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.01079631 Ignoring malparsed backtrace: 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:48.686 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:28.680770+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:48.698 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:48.699 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "started", , 2023-03-30T00:59:48.711 INFO:teuthology.orchestra.run.smithi035.stdout: "client_info": { , 2023-03-30T00:59:48.712 INFO:teuthology.orchestra.run.smithi035.stdout: "client": "client.4555", Ignoring malparsed backtrace: 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 7.2860000000000002e-06 , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:48.724 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0061165480000000003 , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: } , 2023-03-30T00:59:48.737 INFO:teuthology.orchestra.run.smithi035.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:48.833+0000 7efd99558700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:14c33e9f:::smithi20399497-34:3 repair needed (no obc) , 2023-03-30T00:59:48.836 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 0 to 802816 tid 1 , 2023-03-30T00:59:48.838 INFO:tasks.rados.rados.0.smithi203.stdout:331: writing smithi20399497-28 from 802816 to 1228800 tid 2 Ignoring malparsed backtrace: 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:49.306 INFO:teuthology.orchestra.run.smithi035.stdout: "events": [ , 2023-03-30T00:59:49.307 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.0722e-05 , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.323 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.604113+0000", , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.00052637799999999998 , 2023-03-30T00:59:49.341 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0021209829999999999, , 2023-03-30T00:59:49.362 INFO:teuthology.orchestra.run.smithi035.stdout: "type_data": { , 2023-03-30T00:59:49.363 INFO:teuthology.orchestra.run.smithi035.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:16.610251+0000", , 2023-03-30T00:59:49.382 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 8.7749999999999998e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "all_read", , 2023-03-30T00:59:49.408 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:17.608123+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "initiated_at": "2023-03-30T00:59:17.611077+0000", , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "age": 31.691323830999998, , 2023-03-30T00:59:49.422 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0017984959999999999, Ignoring malparsed backtrace: 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:49.436 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "sub_op_started", Ignoring malparsed backtrace: 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 4294967295.9999995 , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:49.450 INFO:teuthology.orchestra.run.smithi035.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.789051+0000", , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0 , 2023-03-30T00:59:49.464 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.796145+0000", , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 0.0040419669999999996 , 2023-03-30T00:59:49.478 INFO:teuthology.orchestra.run.smithi035.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:49.491 INFO:teuthology.orchestra.run.smithi035.stdout: "event": "dispatched", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "time": "2023-03-30T00:59:43.798914+0000", , 2023-03-30T00:59:49.492 INFO:teuthology.orchestra.run.smithi035.stdout: "duration": 1.9406e-05 Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/file.py", line 523, in _write_all , count = self._write(data) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/virtualenv/lib/python3.8/site-packages/paramiko/channel.py", line 1364, in _write Ignoring malparsed backtrace: 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "initiated", , 2023-03-30T00:59:50.032 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.560616+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:50.045 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.583527+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.058 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:50.071 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.2229999999999996e-06 , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.084 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.683455+0000", , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 9.7909999999999999e-06 , 2023-03-30T00:59:50.097 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.680363+0000", , 2023-03-30T00:59:50.110 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.687466+0000", , 2023-03-30T00:59:50.123 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.7453e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:50.135 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T00:59:50.136 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.765837+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:50.148 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.1ds1 77/54 ECSubWrite(tid=239, reqid=client.4555.0:880, at_version=77'66, trim_to=0'0, roll_forward_to=77'64))", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:48.854994+0000", , 2023-03-30T00:59:50.161 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 1.1714130300000001, Ignoring malparsed backtrace: 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.8422000000000001e-05 , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:50.174 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:50.486+0000 7f6a8da31700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 14 (PID: 94707) UID: 0 , 2023-03-30T00:59:50.492 INFO:teuthology.orchestra.run.smithi035.stdout:osd.9: {} , 2023-03-30T00:59:50.513 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} Ignoring malparsed backtrace: 2023-03-30T00:59:51.270 INFO:teuthology.orchestra.run.smithi083.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:51.271 INFO:teuthology.orchestra.run.smithi083.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. , 2023-03-30T00:59:51.280 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops Ignoring malparsed backtrace: 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 1 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:349: finishing write tid 2 to smithi20399497-50 , 2023-03-30T00:59:51.833 INFO:tasks.rados.rados.0.smithi203.stdout:348: finishing copy_from to smithi20399497-15 Ignoring malparsed backtrace: 2023-03-30T00:59:53.011+0000 7f57b74ee700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 13 (PID: 94715) UID: 0 , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument , 2023-03-30T00:59:53.016 INFO:teuthology.orchestra.run.smithi181.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. Ignoring malparsed backtrace: 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:08.599544+0000", , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 7.0852000000000005e-05 , 2023-03-30T00:59:53.343 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:28.684493+0000", , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000121767 , 2023-03-30T00:59:53.366 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.040131732000000003, , 2023-03-30T00:59:53.380 INFO:teuthology.orchestra.run.smithi181.stdout: "type_data": { , 2023-03-30T00:59:53.381 INFO:teuthology.orchestra.run.smithi181.stdout: "flag_point": "commit sent; apply or cleanup", Ignoring malparsed backtrace: 2023-03-30T00:59:53.394 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "all_read", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:33.703140+0000", , 2023-03-30T00:59:53.395 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0410000000000002e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.408 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "started", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:38.756116+0000", , 2023-03-30T00:59:53.409 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 0.000105009 Ignoring malparsed backtrace: 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.422 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "initiated", , 2023-03-30T00:59:53.423 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779826+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "waiting for peered", , 2023-03-30T00:59:53.437 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:42.779943+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.451 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: ] , 2023-03-30T00:59:53.465 INFO:teuthology.orchestra.run.smithi181.stdout: } Ignoring malparsed backtrace: 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0249999999999998e-06 , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: }, , 2023-03-30T00:59:53.479 INFO:teuthology.orchestra.run.smithi181.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: { , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "description": "MOSDECSubOpReadReply(3.9s0 71/42 ECSubReadReply(tid=160, attrs_read=0))", , 2023-03-30T00:59:53.493 INFO:teuthology.orchestra.run.smithi181.stdout: "initiated_at": "2023-03-30T00:59:42.803584+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:53.507 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:43.768289+0000", , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 3.0489999999999999e-06 , 2023-03-30T00:59:53.508 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.521 INFO:teuthology.orchestra.run.smithi181.stdout: "client_addr": "172.21.15.203:0/115282208", , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: "tid": 756 , 2023-03-30T00:59:53.522 INFO:teuthology.orchestra.run.smithi181.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:53.535 INFO:teuthology.orchestra.run.smithi181.stdout: "time": "2023-03-30T00:59:48.845453+0000", , 2023-03-30T00:59:53.536 INFO:teuthology.orchestra.run.smithi181.stdout: "duration": 5.8699999999999997e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:53.843+0000 7fc5039f5700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 8 (PID: 95060) UID: 0 , 2023-03-30T00:59:53.847 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd reweight 13 1 , 2023-03-30T00:59:53.884+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0117e-05 , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:53.977 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.654122+0000", , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:53.990 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.688971+0000", , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0037431610000000001 , 2023-03-30T00:59:54.003 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.015 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.710655+0000", , 2023-03-30T00:59:54.016 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.6818000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.029 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.743870+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.041 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T00:59:54.042 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.754628+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.054 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ , 2023-03-30T00:59:54.067 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.0947e-05 , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.080 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.813138+0000", , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.9743999999999999e-05 , 2023-03-30T00:59:54.093 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0033700029999999999, , 2023-03-30T00:59:54.105 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:54.106 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "reached pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_started", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.824625+0000", , 2023-03-30T00:59:54.118 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.8889999999999999e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000211175 , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.616 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 3.8890000000000002e-05 , 2023-03-30T00:59:54.629 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.630 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 37 , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.643 INFO:teuthology.orchestra.run.smithi203.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T00:59:54.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144504+0000", , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.0872999999999996e-05 , 2023-03-30T00:59:54.666 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.149407+0000", , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.8856000000000001e-05 , 2023-03-30T00:59:54.681 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384053+0000", , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.2444999999999998e-05 , 2023-03-30T00:59:54.695 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 187 , 2023-03-30T00:59:54.714 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.374736+0000", , 2023-03-30T00:59:54.727 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 4.4740000000000002e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384434+0000", , 2023-03-30T00:59:54.741 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.2589000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.581520+0000", , 2023-03-30T00:59:54.758 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000112135 Ignoring malparsed backtrace: 2023-03-30T00:59:54.777 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T00:59:54.778 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578853+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:54.795 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:07.578887+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:54.809 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:12.630036+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.823 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:54.840 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:54.833+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 20=[]:{12=[10]} for object 3:6d0110e8:::smithi20399497-36:c , 2023-03-30T00:59:54.834+0000 7f0f75ef5700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:6563929d:::smithi20399497-15:3 repair needed (no obc) , 2023-03-30T00:59:54.862 DEBUG:teuthology.orchestra.run.smithi203:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.15.asok dump_ops_in_flight Ignoring malparsed backtrace: 2023-03-30T00:59:55.276 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.622353+0000", , 2023-03-30T00:59:55.277 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001925162 Ignoring malparsed backtrace: 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T00:59:55.290 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.302 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T00:59:55.303 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.629137+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T00:59:55.315 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.636040+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.328 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.341 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:55.353 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T00:59:55.354 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.366 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.6981000000000001e-05 , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T00:59:55.379 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.00041520000000000001 , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T00:59:55.392 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:42.809116+0000", , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.8238999999999999e-05 , 2023-03-30T00:59:55.405 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.773072+0000", , 2023-03-30T00:59:55.417 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000102753 , 2023-03-30T00:59:55.418 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:43.786664+0000", , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0077388860000000004 , 2023-03-30T00:59:55.430 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0076006499999999996, , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "type_data": { , 2023-03-30T00:59:55.443 INFO:teuthology.orchestra.run.smithi203.stdout: "flag_point": "started", Ignoring malparsed backtrace: 2023-03-30T00:59:55.455 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:48.827928+0000", , 2023-03-30T00:59:55.456 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 8.1890000000000007e-06 Ignoring malparsed backtrace: 2023-03-30T00:59:56.235 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_historic_ops , 2023-03-30T00:59:56.266 ERROR:teuthology.orchestra.daemon.state:Failed to send signal 1: None , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-30T00:59:56.868+0000 7f287dc5e700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{11=[11,10,f,d],2c=[2c,29,28,27,26,23,20,1e,1c]} for object 3:e7165588:::smithi20399497-16:c , 2023-03-30T00:59:56.870+0000 7fca99663700 -1 log_channel(cluster) log [ERR] : No clone_snaps in snapset 2c=[]:{d=[d],20=[20,1f,1e,1d,1c,1b,17],2c=[2c,29,28,27,26,23]} for object 3:ac746cfe:::smithi20399497-5:c , 2023-03-30T00:59:56.870+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : trim_object: Can not trim 3:bbe97148:::smithi20399497-47:3 repair needed (no obc) Ignoring malparsed backtrace: 2023-03-30T00:59:57.884 INFO:teuthology.orchestra.run.smithi035.stdout:osd.0: {} , 2023-03-30T00:59:57.885+0000 7f7e40f3f700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 2 (PID: 94700) UID: 0 , 2023-03-30T00:59:57.888 INFO:tasks.rados.rados.0.smithi203.stdout:368: writing smithi20399497-3 from 4521984 to 4636672 tid 1 Ignoring malparsed backtrace: 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_change": "24", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend": "0", , 2023-03-30T00:59:58.061 INFO:teuthology.orchestra.run.smithi035.stdout: "last_force_op_resend_prenautilus": "0", Ignoring malparsed backtrace: 2023-03-30T00:59:58.075 INFO:teuthology.orchestra.run.smithi035.stdout: "write_tier": -1, , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "cache_mode": "none", , 2023-03-30T00:59:58.076 INFO:teuthology.orchestra.run.smithi035.stdout: "target_max_bytes": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_grade_decay_rate": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "hit_set_search_last_n": 0, , 2023-03-30T00:59:58.090 INFO:teuthology.orchestra.run.smithi035.stdout: "grade_table": [], Ignoring malparsed backtrace: 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "up_thru": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "down_at": 0, , 2023-03-30T00:59:58.110 INFO:teuthology.orchestra.run.smithi035.stdout: "lost_at": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.129 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.130 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", Ignoring malparsed backtrace: 2023-03-30T00:59:58.146 INFO:teuthology.orchestra.run.smithi035.stdout: "addrvec": [ , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.147 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", Ignoring malparsed backtrace: 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "uuid": "32fd7d07-dd9a-476c-9fc4-da8e1f2855c0", , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "up": 1, , 2023-03-30T00:59:58.163 INFO:teuthology.orchestra.run.smithi035.stdout: "in": 1, Ignoring malparsed backtrace: 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.177 INFO:teuthology.orchestra.run.smithi035.stdout: "cluster_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.190 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6807", Ignoring malparsed backtrace: 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addr": "172.21.15.35:6809/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_front_addr": "172.21.15.35:6807/95087", , 2023-03-30T00:59:58.204 INFO:teuthology.orchestra.run.smithi035.stdout: "state": [ Ignoring malparsed backtrace: 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.181:6816", , 2023-03-30T00:59:58.217 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94712 Ignoring malparsed backtrace: 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: ] , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: }, , 2023-03-30T00:59:58.230 INFO:teuthology.orchestra.run.smithi035.stdout: "heartbeat_back_addrs": { Ignoring malparsed backtrace: 2023-03-30T00:59:58.243 INFO:teuthology.orchestra.run.smithi035.stdout: { , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v1", , 2023-03-30T00:59:58.244 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.35:6823", Ignoring malparsed backtrace: 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_begin": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "last_clean_end": 0, , 2023-03-30T00:59:58.257 INFO:teuthology.orchestra.run.smithi035.stdout: "up_from": 22, Ignoring malparsed backtrace: 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "type": "v2", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "addr": "172.21.15.203:6810", , 2023-03-30T00:59:58.270 INFO:teuthology.orchestra.run.smithi035.stdout: "nonce": 94976 Ignoring malparsed backtrace: 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "osd": 5, , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.283 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "down_stamp": "0.000000", , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_probability": 0, , 2023-03-30T00:59:58.296 INFO:teuthology.orchestra.run.smithi035.stdout: "laggy_interval": 0, Ignoring malparsed backtrace: 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout: "num_ops": 0 , 2023-03-30T00:59:58.311 INFO:teuthology.orchestra.run.smithi083.stdout:} , 2023-03-30T00:59:58.312 INFO:tasks.thrashosds.thrasher:j is {'epoch': 86, 'fsid': '8f2049d8-2bb0-46fe-a814-ce26c43a7f40', 'created': '2023-03-30T00:58:33.472724+0000', 'modified': '2023-03-30T00:59:57.844713+0000', 'last_up_change': '2023-03-30T00:59:43.755794+0000', 'last_in_change': '2023-03-30T00:58:39.154518+0000', 'flags': 'sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit', 'flags_num': 5799936, 'flags_set': ['pglog_hardlimit', 'purged_snapdirs', 'recovery_deletes', 'sortbitwise'], 'crush_version': 7, 'full_ratio': 0.949999988079071, 'backfillfull_ratio': 0.8999999761581421, 'nearfull_ratio': 0.8500000238418579, 'cluster_snapshot': '', 'pool_max': 3, 'max_osd': 16, 'require_min_compat_client': 'luminous', 'min_compat_client': 'jewel', 'require_osd_release': 'reef', 'allow_crimson': False, 'pools': [{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-03-30T00:58:45.356303+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '24', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 15.380000114440918, 'score_stable': 15.380000114440918, 'optimal_score': 0.12999999523162842, 'raw_score_acting': 2, 'raw_score_stable': 2, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 2, 'pool_name': 'rbd', 'create_time': '2023-03-30T00:58:49.826143+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 2, 'min_size': 1, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 8, 'pg_placement_num': 8, 'pg_placement_num_target': 8, 'pg_num_target': 8, 'pg_num_pending': 8, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '27', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rbd': {}}, 'read_balance': {'score_acting': 3.7699999809265137, 'score_stable': 3.7699999809265137, 'optimal_score': 0.7300000190734863, 'raw_score_acting': 2.75, 'raw_score_stable': 2.75, 'primary_affinity_weighted': 0.9375, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 0.9399999976158142}}, {'pool': 3, 'pool_name': 'unique_pool_0', 'create_time': '2023-03-30T00:59:00.541501+0000', 'flags': 8193, 'flags_names': 'hashpspool,selfmanaged_snaps', 'type': 3, 'size': 7, 'min_size': 5, 'crush_rule': 1, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 36, 'pg_placement_num': 16, 'pg_placement_num_target': 36, 'pg_num_target': 36, 'pg_num_pending': 36, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '86', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '54', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 54, 'snap_epoch': 86, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': 'shecprofile', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 16384, 'expected_num_objects': 0, 'fast_read': False, 'options': {}, 'application_metadata': {'rados': {}}}], 'osds': [{'osd': 0, 'uuid': 'ebbb26ab-20a8-4846-8b3f-e2fbabb7cf75', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 42, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6810', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6811', 'nonce': 95086}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6812', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6813', 'nonce': 95086}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6816', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6817', 'nonce': 95086}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6814', 'nonce': 95086}, {'type': 'v1', 'addr': '172.21.15.35:6815', 'nonce': 95086}]}, 'public_addr': '172.21.15.35:6811/95086', 'cluster_addr': '172.21.15.35:6813/95086', 'heartbeat_back_addr': '172.21.15.35:6817/95086', 'heartbeat_front_addr': '172.21.15.35:6815/95086', 'state': ['exists', 'up']}, {'osd': 1, 'uuid': '3bc34814-68b2-4d01-8249-953700168560', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6800', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6801', 'nonce': 94718}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6802', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6803', 'nonce': 94718}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6806', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6807', 'nonce': 94718}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6804', 'nonce': 94718}, {'type': 'v1', 'addr': '172.21.15.83:6805', 'nonce': 94718}]}, 'public_addr': '172.21.15.83:6801/94718', 'cluster_addr': '172.21.15.83:6803/94718', 'heartbeat_back_addr': '172.21.15.83:6807/94718', 'heartbeat_front_addr': '172.21.15.83:6805/94718', 'state': ['exists', 'up']}, {'osd': 2, 'uuid': '76f71867-1d44-4527-877a-fd92aac57333', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6808', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6809', 'nonce': 94710}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6810', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6811', 'nonce': 94710}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6814', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6815', 'nonce': 94710}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6812', 'nonce': 94710}, {'type': 'v1', 'addr': '172.21.15.181:6813', 'nonce': 94710}]}, 'public_addr': '172.21.15.181:6809/94710', 'cluster_addr': '172.21.15.181:6811/94710', 'heartbeat_back_addr': '172.21.15.181:6815/94710', 'heartbeat_front_addr': '172.21.15.181:6813/94710', 'state': ['exists', 'up']}, {'osd': 3, 'uuid': '954b6f1d-986f-49e6-93e5-d3423f3bf421', 'up': 0, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 22, 'last_clean_end': 27, 'up_from': 70, 'up_thru': 0, 'down_at': 72, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6816', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6817', 'nonce': 100463}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6818', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6819', 'nonce': 100463}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6822', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6823', 'nonce': 100463}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6820', 'nonce': 100463}, {'type': 'v1', 'addr': '172.21.15.203:6821', 'nonce': 100463}]}, 'public_addr': '172.21.15.203:6817/100463', 'cluster_addr': '172.21.15.203:6819/100463', 'heartbeat_back_addr': '172.21.15.203:6823/100463', 'heartbeat_front_addr': '172.21.15.203:6821/100463', 'state': ['exists']}, {'osd': 4, 'uuid': 'ad653e95-e58a-4fec-9db8-d313042c2e97', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6826', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6827', 'nonce': 95091}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6828', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6829', 'nonce': 95091}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6832', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6833', 'nonce': 95091}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6830', 'nonce': 95091}, {'type': 'v1', 'addr': '172.21.15.35:6831', 'nonce': 95091}]}, 'public_addr': '172.21.15.35:6827/95091', 'cluster_addr': '172.21.15.35:6829/95091', 'heartbeat_back_addr': '172.21.15.35:6833/95091', 'heartbeat_front_addr': '172.21.15.35:6831/95091', 'state': ['exists', 'up']}, {'osd': 5, 'uuid': '32fd7d07-dd9a-476c-9fc4-da8e1f2855c0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6808', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6809', 'nonce': 94717}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6810', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6811', 'nonce': 94717}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6814', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6815', 'nonce': 94717}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6812', 'nonce': 94717}, {'type': 'v1', 'addr': '172.21.15.83:6813', 'nonce': 94717}]}, 'public_addr': '172.21.15.83:6809/94717', 'cluster_addr': '172.21.15.83:6811/94717', 'heartbeat_back_addr': '172.21.15.83:6815/94717', 'heartbeat_front_addr': '172.21.15.83:6813/94717', 'state': ['exists', 'up']}, {'osd': 6, 'uuid': '28a11dac-0c35-4a42-8250-cfedff301ba0', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 0, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6800', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6801', 'nonce': 94711}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6802', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6803', 'nonce': 94711}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6806', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6807', 'nonce': 94711}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6804', 'nonce': 94711}, {'type': 'v1', 'addr': '172.21.15.181:6805', 'nonce': 94711}]}, 'public_addr': '172.21.15.181:6801/94711', 'cluster_addr': '172.21.15.181:6803/94711', 'heartbeat_back_addr': '172.21.15.181:6807/94711', 'heartbeat_front_addr': '172.21.15.181:6805/94711', 'state': ['exists', 'up']}, {'osd': 7, 'uuid': 'ea60440a-7f79-4503-9ddb-b513c366c68a', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6800', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6801', 'nonce': 94975}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6802', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6803', 'nonce': 94975}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6806', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6807', 'nonce': 94975}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6804', 'nonce': 94975}, {'type': 'v1', 'addr': '172.21.15.203:6805', 'nonce': 94975}]}, 'public_addr': '172.21.15.203:6801/94975', 'cluster_addr': '172.21.15.203:6803/94975', 'heartbeat_back_addr': '172.21.15.203:6807/94975', 'heartbeat_front_addr': '172.21.15.203:6805/94975', 'state': ['exists', 'up']}, {'osd': 8, 'uuid': '2614c844-ce70-4875-86cd-7ce40b5002db', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 21, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6802', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6803', 'nonce': 95087}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6804', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6805', 'nonce': 95087}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6808', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6809', 'nonce': 95087}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6806', 'nonce': 95087}, {'type': 'v1', 'addr': '172.21.15.35:6807', 'nonce': 95087}]}, 'public_addr': '172.21.15.35:6803/95087', 'cluster_addr': '172.21.15.35:6805/95087', 'heartbeat_back_addr': '172.21.15.35:6809/95087', 'heartbeat_front_addr': '172.21.15.35:6807/95087', 'state': ['exists', 'up']}, {'osd': 9, 'uuid': '110531cf-e218-4332-aef0-9f05d71178bb', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 25, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6817', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6819', 'nonce': 94719}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6821', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6823', 'nonce': 94719}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6829', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6831', 'nonce': 94719}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6825', 'nonce': 94719}, {'type': 'v1', 'addr': '172.21.15.83:6827', 'nonce': 94719}]}, 'public_addr': '172.21.15.83:6819/94719', 'cluster_addr': '172.21.15.83:6823/94719', 'heartbeat_back_addr': '172.21.15.83:6831/94719', 'heartbeat_front_addr': '172.21.15.83:6827/94719', 'state': ['exists', 'up']}, {'osd': 10, 'uuid': 'c7b567e6-1561-4f29-bbd4-65a6a4746929', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6816', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6817', 'nonce': 94712}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6818', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6819', 'nonce': 94712}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6825', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6827', 'nonce': 94712}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6821', 'nonce': 94712}, {'type': 'v1', 'addr': '172.21.15.181:6823', 'nonce': 94712}]}, 'public_addr': '172.21.15.181:6817/94712', 'cluster_addr': '172.21.15.181:6819/94712', 'heartbeat_back_addr': '172.21.15.181:6827/94712', 'heartbeat_front_addr': '172.21.15.181:6823/94712', 'state': ['exists', 'up']}, {'osd': 11, 'uuid': '36e97302-d025-4aaf-ade4-3ece93f0ffd6', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6824', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6825', 'nonce': 94977}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6826', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6827', 'nonce': 94977}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6830', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6831', 'nonce': 94977}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6828', 'nonce': 94977}, {'type': 'v1', 'addr': '172.21.15.203:6829', 'nonce': 94977}]}, 'public_addr': '172.21.15.203:6825/94977', 'cluster_addr': '172.21.15.203:6827/94977', 'heartbeat_back_addr': '172.21.15.203:6831/94977', 'heartbeat_front_addr': '172.21.15.203:6829/94977', 'state': ['exists', 'up']}, {'osd': 12, 'uuid': 'bb2a234d-18d7-4ed3-b250-573eeb31cc0b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 0, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6818', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6819', 'nonce': 95092}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6820', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6821', 'nonce': 95092}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6824', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6825', 'nonce': 95092}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.35:6822', 'nonce': 95092}, {'type': 'v1', 'addr': '172.21.15.35:6823', 'nonce': 95092}]}, 'public_addr': '172.21.15.35:6819/95092', 'cluster_addr': '172.21.15.35:6821/95092', 'heartbeat_back_addr': '172.21.15.35:6825/95092', 'heartbeat_front_addr': '172.21.15.35:6823/95092', 'state': ['exists', 'up']}, {'osd': 13, 'uuid': '34d7ae22-e68e-4faa-b647-b251e0b7336b', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6816', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6818', 'nonce': 94720}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6820', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6822', 'nonce': 94720}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6828', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6830', 'nonce': 94720}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.83:6824', 'nonce': 94720}, {'type': 'v1', 'addr': '172.21.15.83:6826', 'nonce': 94720}]}, 'public_addr': '172.21.15.83:6818/94720', 'cluster_addr': '172.21.15.83:6822/94720', 'heartbeat_back_addr': '172.21.15.83:6830/94720', 'heartbeat_front_addr': '172.21.15.83:6826/94720', 'state': ['exists', 'up']}, {'osd': 14, 'uuid': '72801aa3-e45b-4a6f-9f52-d327b2caa7aa', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 72, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6820', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6822', 'nonce': 94713}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6824', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6826', 'nonce': 94713}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6830', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6831', 'nonce': 94713}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.181:6828', 'nonce': 94713}, {'type': 'v1', 'addr': '172.21.15.181:6829', 'nonce': 94713}]}, 'public_addr': '172.21.15.181:6822/94713', 'cluster_addr': '172.21.15.181:6826/94713', 'heartbeat_back_addr': '172.21.15.181:6831/94713', 'heartbeat_front_addr': '172.21.15.181:6829/94713', 'state': ['exists', 'up']}, {'osd': 15, 'uuid': '99ba406f-ecd6-46eb-bddb-66ae54f63dcf', 'up': 1, 'in': 1, 'weight': 1, 'primary_affinity': 1, 'last_clean_begin': 0, 'last_clean_end': 0, 'up_from': 22, 'up_thru': 54, 'down_at': 0, 'lost_at': 0, 'public_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6808', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6809', 'nonce': 94976}]}, 'cluster_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6810', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6811', 'nonce': 94976}]}, 'heartbeat_back_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6814', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6815', 'nonce': 94976}]}, 'heartbeat_front_addrs': {'addrvec': [{'type': 'v2', 'addr': '172.21.15.203:6812', 'nonce': 94976}, {'type': 'v1', 'addr': '172.21.15.203:6813', 'nonce': 94976}]}, 'public_addr': '172.21.15.203:6809/94976', 'cluster_addr': '172.21.15.203:6811/94976', 'heartbeat_back_addr': '172.21.15.203:6815/94976', 'heartbeat_front_addr': '172.21.15.203:6813/94976', 'state': ['exists', 'up']}], 'osd_xinfo': [{'osd': 0, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 1, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 2, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 3, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 72}, {'osd': 4, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 5, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 6, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 7, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 8, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 9, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 10, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 11, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 12, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 13, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 14, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}, {'osd': 15, 'down_stamp': '0.000000', 'laggy_probability': 0, 'laggy_interval': 0, 'features': 4540138322906710015, 'old_weight': 0, 'last_purged_snaps_scrub': '0.000000', 'dead_epoch': 0}], 'pg_upmap': [], 'pg_upmap_items': [], 'pg_upmap_primaries': [], 'pg_temp': [], 'primary_temp': [], 'blocklist': {}, 'range_blocklist': {}, 'erasure_code_profiles': {'default': {'crush-failure-domain': 'osd', 'k': '2', 'm': '1', 'plugin': 'jerasure', 'technique': 'reed_sol_van'}, 'shecprofile': {'c': '2', 'crush-device-class': '', 'crush-failure-domain': 'osd', 'crush-root': 'default', 'k': '4', 'm': '3', 'name': 'shecprofile', 'plugin': 'shec', 'technique': 'multiple'}}, 'removed_snaps_queue': [{'pool': 3, 'snaps': [{'begin': 2, 'length': 2}, {'begin': 5, 'length': 2}, {'begin': 10, 'length': 1}, {'begin': 12, 'length': 2}, {'begin': 15, 'length': 4}, {'begin': 23, 'length': 2}, {'begin': 27, 'length': 1}, {'begin': 29, 'length': 1}, {'begin': 31, 'length': 1}, {'begin': 34, 'length': 2}, {'begin': 39, 'length': 1}]}], 'new_removed_snaps': [], 'new_purged_snaps': [], 'crush_node_flags': {}, 'device_class_flags': {}, 'stretch_mode': {'stretch_mode_enabled': False, 'stretch_bucket_count': 0, 'degraded_stretch_mode': 0, 'recovering_stretch_mode': 0, 'stretch_mode_bucket': 0}} Ignoring malparsed backtrace: 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:2f snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {27, 28, 29, 2c, 2d, 2e, 2f} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:26 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {22, 23, 26} ...repaired , 2023-03-30T00:59:58.476+0000 7f5797899700 -1 log_channel(cluster) log [ERR] : osd.13(0) found snap mapper error on pg 3.1ds0> oid 3:b8d697e8:::smithi20399497-37:20 snaps in mapper: {d, 10, 12, 17, 1b, 1c, 1d, 1e, 1f, 20, 22, 23, 26, 27, 28, 29, 2c, 2d, 2e, 2f, 32, 33, 35, 36}, oi: {1e, 1f, 20} ...repaired Ignoring malparsed backtrace: 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.712749+0000", , 2023-03-30T00:59:58.595 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 Ignoring malparsed backtrace: 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.609 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:58.623 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.681262+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.637 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", Ignoring malparsed backtrace: 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "reached pg", , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ , 2023-03-30T00:59:58.651 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.1990999999999999e-05 , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.665 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.713837+0000", , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 1.0708e-05 , 2023-03-30T00:59:58.679 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.806010+0000", , 2023-03-30T00:59:58.693 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.814067+0000", , 2023-03-30T00:59:58.707 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.3025000000000001e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T00:59:58.721 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.858217+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:58.735 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", Ignoring malparsed backtrace: 2023-03-30T00:59:58.864 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph osd set noscrub , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:376: writing smithi20399497-48 from 2457600 to 2621440 tid 1 , 2023-03-30T00:59:58.873 INFO:tasks.rados.rados.0.smithi203.stdout:377: rmattr oid 19 current snap is 33 Ignoring malparsed backtrace: 2023-03-30T00:59:59.089 INFO:tasks.rados.rados.0.smithi203.stdout:update_object_version oid 16 v 50 (ObjNum 134 snap 26 seq_num 134) dirty exists , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 1 to smithi20399497-17 , 2023-03-30T00:59:59.090 INFO:tasks.rados.rados.0.smithi203.stdout:385: finishing write tid 2 to smithi20399497-17 Ignoring malparsed backtrace: 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0011711810000000001 , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.256 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "flag_point": "commit sent; apply or cleanup", , 2023-03-30T00:59:59.272 INFO:teuthology.orchestra.run.smithi083.stdout: "client_info": { Ignoring malparsed backtrace: 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606447+0000", , 2023-03-30T00:59:59.285 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.3420000000000001e-06 , 2023-03-30T00:59:59.286 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.613875+0000", , 2023-03-30T00:59:59.298 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0042258850000000004 , 2023-03-30T00:59:59.299 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:23.691580+0000", , 2023-03-30T00:59:59.311 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999971 Ignoring malparsed backtrace: 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:26.688990+0000", , 2023-03-30T00:59:59.324 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.0050633730000000004 Ignoring malparsed backtrace: 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.337 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732738+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.350 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.363 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.376 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.9600000000000006e-06 , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.389 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001601318 , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.402 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.830761+0000", , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.415 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.428 INFO:teuthology.orchestra.run.smithi083.stdout:} Ignoring malparsed backtrace: 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.569096+0000", , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.999999 , 2023-03-30T00:59:59.884 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.902 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "MOSDECSubOpWrite(3.6s5 36/30 ECSubWrite(tid=17, reqid=client.4555.0:186, at_version=36'9, trim_to=0'0, roll_forward_to=33'6))", Ignoring malparsed backtrace: 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:07.598900+0000", , 2023-03-30T00:59:59.917 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4.8109999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "header_read", , 2023-03-30T00:59:59.932 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.603403+0000", Ignoring malparsed backtrace: 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T00:59:59.947 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T00:59:59.962 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T00:59:59.986 INFO:teuthology.orchestra.run.smithi083.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.4739000000000001e-05 , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.000 INFO:teuthology.orchestra.run.smithi083.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:32.732868+0000", , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 8.4030000000000007e-06 , 2023-03-30T01:00:00.021 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.034 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "throttled", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:38.760206+0000", , 2023-03-30T01:00:00.035 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:00.047 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "done", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.845424+0000", , 2023-03-30T01:00:00.048 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.2374999999999998e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:00.060 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.854031+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.214 DEBUG:teuthology.orchestra.run.smithi083:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 30 ceph --cluster ceph --admin-daemon /var/run/ceph/ceph-osd.13.asok dump_blocked_ops , 2023-03-30T01:00:00.274+0000 7f6b68ab7700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 7 (PID: 94966) UID: 0 , 2023-03-30T01:00:00.320 DEBUG:teuthology.orchestra.run.smithi035:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false Ignoring malparsed backtrace: 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:175 3.as0 3:5d617aee:::smithi20399497-20:head [stat] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e36)", , 2023-03-30T01:00:00.491 INFO:teuthology.orchestra.run.smithi083.stdout: "initiated_at": "2023-03-30T00:59:07.550221+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "tid": 388 , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.504 INFO:teuthology.orchestra.run.smithi083.stdout: "events": [ Ignoring malparsed backtrace: 2023-03-30T01:00:00.516 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:16.606225+0000", , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 4294967295.9999995 , 2023-03-30T01:00:00.517 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:28.673419+0000", , 2023-03-30T01:00:00.529 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 2.0645e-05 , 2023-03-30T01:00:00.530 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:33.703355+0000", , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 7.6669999999999996e-06 , 2023-03-30T01:00:00.542 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.555 INFO:teuthology.orchestra.run.smithi083.stdout: "description": "osd_op(client.4555.0:705 3.fs0 3:f76751ac:::smithi20399497-42:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e67)", Ignoring malparsed backtrace: 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.800579+0000", , 2023-03-30T01:00:00.568 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.00046025699999999999 Ignoring malparsed backtrace: 2023-03-30T01:00:00.580 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "sub_op_started", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.805906+0000", , 2023-03-30T01:00:00.581 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.001011241 Ignoring malparsed backtrace: 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "age": 17.667686744000001, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "duration": 0.014015517, , 2023-03-30T01:00:00.593 INFO:teuthology.orchestra.run.smithi083.stdout: "type_data": { Ignoring malparsed backtrace: 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", , 2023-03-30T01:00:00.606 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:42.808867+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "started", , 2023-03-30T01:00:00.619 INFO:teuthology.orchestra.run.smithi083.stdout: "time": "2023-03-30T00:59:48.811121+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:00.631 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.632 INFO:teuthology.orchestra.run.smithi083.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.644 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: }, , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: { , 2023-03-30T01:00:00.657 INFO:teuthology.orchestra.run.smithi083.stdout: "event": "reached_pg", Ignoring malparsed backtrace: 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: ] , 2023-03-30T01:00:00.670 INFO:teuthology.orchestra.run.smithi083.stdout: } Ignoring malparsed backtrace: 2023-03-30T01:00:00.726 INFO:teuthology.orchestra.run.smithi035.stdout:osd.13: {} , 2023-03-30T01:00:00.746 INFO:teuthology.orchestra.run.smithi035.stdout:osd.14: {} , 2023-03-30T01:00:00.767 INFO:teuthology.orchestra.run.smithi035.stdout:osd.15: {} Ignoring malparsed backtrace: OSError: Socket is closed , 2023-03-30T01:00:01.589+0000 7f5cfc24b700 -1 received signal: Hangup from /usr/bin/python3 /bin/daemon-helper kill ceph-osd -f --cluster ceph -i 10 (PID: 94706) UID: 0 , 2023-03-30T01:00:01.614 INFO:teuthology.orchestra.run.smithi181.stdout:ERROR: (22) Invalid argument Ignoring malparsed backtrace: 2023-03-30T01:00:03.108 INFO:teuthology.orchestra.run.smithi035.stdout:osd.10: {} , 2023-03-30T01:00:03.128 INFO:teuthology.orchestra.run.smithi035.stdout:osd.11: {} , 2023-03-30T01:00:03.133 INFO:teuthology.orchestra.run.smithi203.stdout:{ Ignoring malparsed backtrace: 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "queued_for_pg", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.332416+0000", , 2023-03-30T01:00:03.461 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1637999999999999e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.479 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.812171+0000", , 2023-03-30T01:00:03.480 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 6.3927e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.500 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:47.820021+0000", , 2023-03-30T01:00:03.501 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 2.446e-05 Ignoring malparsed backtrace: 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.514 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", , 2023-03-30T01:00:03.515 INFO:teuthology.orchestra.run.smithi203.stdout: "tid": 140 Ignoring malparsed backtrace: 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", , 2023-03-30T01:00:03.528 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.144542+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.541 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", , 2023-03-30T01:00:03.542 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.381561+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", , 2023-03-30T01:00:03.559 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:58:48.384196+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_info": { , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client": "client.4232", , 2023-03-30T01:00:03.572 INFO:teuthology.orchestra.run.smithi203.stdout: "client_addr": "172.21.15.35:0/1450367353", Ignoring malparsed backtrace: 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.585 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "all_read", Ignoring malparsed backtrace: 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.598 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:03.611 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:03.612 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", Ignoring malparsed backtrace: 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.625 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 5.5550000000000002e-06 , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:03.638 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.000104752 , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: } , 2023-03-30T01:00:03.652 INFO:teuthology.orchestra.run.smithi203.stdout: ] Ignoring malparsed backtrace: 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:32.705642+0000", , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:03.665 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.759581+0000", , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 1.1364e-05 , 2023-03-30T01:00:03.678 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0038820090000000001 , 2023-03-30T01:00:04.137 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.138 INFO:teuthology.orchestra.run.smithi203.stdout: { Ignoring malparsed backtrace: 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.623852+0000", , 2023-03-30T01:00:04.152 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 , 2023-03-30T01:00:04.153 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.628576+0000", , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 7.4080000000000001e-06 , 2023-03-30T01:00:04.173 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.191 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:18.635982+0000", , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.0049630509999999996 , 2023-03-30T01:00:04.192 INFO:teuthology.orchestra.run.smithi203.stdout: }, Ignoring malparsed backtrace: 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "throttled", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.721032+0000", , 2023-03-30T01:00:04.211 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0 Ignoring malparsed backtrace: 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.724382+0000", , 2023-03-30T01:00:04.229 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 9.4830000000000005e-06 Ignoring malparsed backtrace: 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "sub_op_committed", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.731143+0000", , 2023-03-30T01:00:04.247 INFO:teuthology.orchestra.run.smithi203.stdout: "duration": 0.001987801 Ignoring malparsed backtrace: 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "initiated", , 2023-03-30T01:00:04.265 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:23.726052+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "reached_pg", , 2023-03-30T01:00:04.278 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:33.738720+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "done", , 2023-03-30T01:00:04.291 INFO:teuthology.orchestra.run.smithi203.stdout: "time": "2023-03-30T00:59:38.767733+0000", Ignoring malparsed backtrace: 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.304 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "dispatched", Ignoring malparsed backtrace: 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.317 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "started", Ignoring malparsed backtrace: 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: }, , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: { , 2023-03-30T01:00:04.330 INFO:teuthology.orchestra.run.smithi203.stdout: "event": "commit_sent", Ignoring malparsed backtrace: 2023-03-30T01:00:04.342 INFO:teuthology.orchestra.run.smithi203.stdout: "description": "osd_op(client.4555.0:780 3.13s0 3:cbc5c355:::smithi20399497-3:head [read 0~1] snapc 0=[] ondisk+read+rwordered+known_if_redirected+supports_pool_eio e72)", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "initiated_at": "2023-03-30T00:59:43.841834+0000", , 2023-03-30T01:00:04.343 INFO:teuthology.orchestra.run.smithi203.stdout: "age": 20.282786238, Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f8ac729ece0] , 2: gsignal() Ignoring malparsed backtrace: File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 98, in __next__ , resurrect_traceback(result) , File "/home/teuthworker/src/git.ceph.com_teuthology_e3729161eecf8e6d09a4069acbb883cdcd5e9f50/teuthology/parallel.py", line 30, in resurrect_traceback Misunderstood line: Missing teuthology log /home/teuthworker/archive/benhanokh-2023-03-28_22:46:34-rados-WIP_GBH_snap_mapper_B-distro-default-smithi/7224388/teuthology.log Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Ignoring malparsed backtrace: ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) , 1: /lib64/libpthread.so.0(+0x12cf0) [0x7f72aa6e6cf0] , 2: gsignal() Ignoring malparsed backtrace: dumped all , 2023-03-29T23:32:45.591 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":98,"stamp":"2023-03-29T23:32:44.891912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002204"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117001,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.222}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182411,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.248}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:32:45.593 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:08.913 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":110,"stamp":"2023-03-29T23:33:08.896703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002302"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117006,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182415,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:08.914 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:33:24.473+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:33:25.324+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:33:25.485+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:33:46.656 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2023-03-29T23:33:44.904146+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002412"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.960281+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.960281+0000","last_peered":"2023-03-29T23:32:23.960281+0000","last_clean":"2023-03-29T23:32:23.960281+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.960281+0000","last_undegraded":"2023-03-29T23:32:23.960281+0000","last_fullsized":"2023-03-29T23:32:23.960281+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958769+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:22.958769+0000","last_peered":"2023-03-29T23:32:22.958769+0000","last_clean":"2023-03-29T23:32:22.958769+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:22.958769+0000","last_undegraded":"2023-03-29T23:32:22.958769+0000","last_fullsized":"2023-03-29T23:32:22.958769+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958945+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:22.958945+0000","last_peered":"2023-03-29T23:32:22.958945+0000","last_clean":"2023-03-29T23:32:22.958945+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:22.958945+0000","last_undegraded":"2023-03-29T23:32:22.958945+0000","last_fullsized":"2023-03-29T23:32:22.958945+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":27,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.956786+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:22.956786+0000","last_peered":"2023-03-29T23:32:22.956786+0000","last_clean":"2023-03-29T23:32:22.956786+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:22.956786+0000","last_undegraded":"2023-03-29T23:32:22.956786+0000","last_fullsized":"2023-03-29T23:32:22.956786+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.963311+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.963311+0000","last_peered":"2023-03-29T23:32:23.963311+0000","last_clean":"2023-03-29T23:32:23.963311+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.963311+0000","last_undegraded":"2023-03-29T23:32:23.963311+0000","last_fullsized":"2023-03-29T23:32:23.963311+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":26,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.958712+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:22.958712+0000","last_peered":"2023-03-29T23:32:22.958712+0000","last_clean":"2023-03-29T23:32:22.958712+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:22.958712+0000","last_undegraded":"2023-03-29T23:32:22.958712+0000","last_fullsized":"2023-03-29T23:32:22.958712+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117014,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:33:01 2023","interfaces":[{"interface":"back","average":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.16,"5min":0.16,"15min":0.16},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.22600000000000001},{"interface":"front","average":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"last":0.25700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182423,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.501,"5min":0.501,"15min":0.501},"last":0.23699999999999999},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.20399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:33:46.657 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:34:02.682+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:03.586+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:34:03.702+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:34:23.797+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:34:23.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:34:44.376 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":257,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":8,"modified":"2023-03-29T23:34:16.910333+0000","services":{}},"progress_events":{}} , 2023-03-29T23:34:44.390 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:35:01.564 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:35:01.577 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":166,"stamp":"2023-03-29T23:35:00.917301+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001752"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958893+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:32:23.958893+0000","last_peered":"2023-03-29T23:32:23.958893+0000","last_clean":"2023-03-29T23:32:23.958893+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:32:23.958893+0000","last_undegraded":"2023-03-29T23:32:23.958893+0000","last_fullsized":"2023-03-29T23:32:23.958893+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:24.885630+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:32:24.885630+0000","last_peered":"2023-03-29T23:32:24.885630+0000","last_clean":"2023-03-29T23:32:24.885630+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:32:24.885630+0000","last_undegraded":"2023-03-29T23:32:24.885630+0000","last_fullsized":"2023-03-29T23:32:24.885630+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117030,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.185,"5min":0.16,"15min":0.16},"max":{"1min":0.41099999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20599999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.20000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182438,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:34:05 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28799999999999998,"15min":0.29199999999999998},"min":{"1min":0.187,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.42599999999999999,"5min":0.501,"15min":0.501},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.28299999999999997,"5min":0.28799999999999998,"15min":0.28899999999999998},"min":{"1min":0.20399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.44700000000000001,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.439}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:35:18.501 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:35:18.829 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:35:33.417+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:35:34.064+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:35:34.371+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:35:53.002 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:35:53.002 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:36:10.297 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:36:10.311 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":200,"stamp":"2023-03-29T23:36:08.929811+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002120"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.962800+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:32:23.962800+0000","last_peered":"2023-03-29T23:32:23.962800+0000","last_clean":"2023-03-29T23:32:23.962800+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:32:23.962800+0000","last_undegraded":"2023-03-29T23:32:23.962800+0000","last_fullsized":"2023-03-29T23:32:23.962800+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958870+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:32:23.958870+0000","last_peered":"2023-03-29T23:32:23.958870+0000","last_clean":"2023-03-29T23:32:23.958870+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:32:23.958870+0000","last_undegraded":"2023-03-29T23:32:23.958870+0000","last_fullsized":"2023-03-29T23:32:23.958870+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.968183+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:32:23.968183+0000","last_peered":"2023-03-29T23:32:23.968183+0000","last_clean":"2023-03-29T23:32:23.968183+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:32:23.968183+0000","last_undegraded":"2023-03-29T23:32:23.968183+0000","last_fullsized":"2023-03-29T23:32:23.968183+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":27,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:32:23.958865+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:32:23.958865+0000","last_peered":"2023-03-29T23:32:23.958865+0000","last_clean":"2023-03-29T23:32:23.958865+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:32:23.958865+0000","last_undegraded":"2023-03-29T23:32:23.958865+0000","last_fullsized":"2023-03-29T23:32:23.958865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117043,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27500000000000002,"15min":0.27300000000000002},"min":{"1min":0.223,"5min":0.16,"15min":0.16},"max":{"1min":0.35599999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.27900000000000003,"15min":0.27700000000000002},"min":{"1min":0.20000000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.375,"5min":0.42399999999999999,"15min":0.42399999999999999},"last":0.21199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182451,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:35:06 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28199999999999997,"15min":0.28999999999999998},"min":{"1min":0.19600000000000001,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.40799999999999997,"5min":0.501,"15min":0.501},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.27300000000000002,"5min":0.28499999999999998,"15min":0.28799999999999998},"min":{"1min":0.19800000000000001,"5min":0.193,"15min":0.193},"max":{"1min":0.439,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:36:26.144+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:36:26.197+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:36:27.148+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:36:45.356 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-03-29T23:36:44.937114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002601"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:59.969534+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:33:59.969534+0000","last_peered":"2023-03-29T23:33:59.969534+0000","last_clean":"2023-03-29T23:33:59.969534+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:33:59.969534+0000","last_undegraded":"2023-03-29T23:33:59.969534+0000","last_fullsized":"2023-03-29T23:33:59.969534+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:33:58.954034+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:33:58.954034+0000","last_peered":"2023-03-29T23:33:58.954034+0000","last_clean":"2023-03-29T23:33:58.954034+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:33:58.954034+0000","last_undegraded":"2023-03-29T23:33:58.954034+0000","last_fullsized":"2023-03-29T23:33:58.954034+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117050,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.255,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17499999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.17599999999999999},{"interface":"front","average":{"1min":0.25600000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.19}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182459,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:36:07 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.27500000000000002,"15min":0.28699999999999998},"min":{"1min":0.189,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":0.41499999999999998,"5min":0.501,"15min":0.501},"last":0.317},{"interface":"front","average":{"1min":0.26200000000000001,"5min":0.27900000000000003,"15min":0.28599999999999998},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.40200000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:36:45.357 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} Ignoring malparsed backtrace: 2023-03-29T23:37:04.018+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:04.085+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:37:05.038+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:37:22.460 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:37:22.460 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:37:22.848 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:37:40.882+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:37:41.349 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:37:59.183 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:37:59.184 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":452,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":9,"modified":"2023-03-29T23:37:18.943155+0000","services":{}},"progress_events":{}} , 2023-03-29T23:37:59.198 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: dumped all , 2023-03-29T23:38:16.402 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":263,"stamp":"2023-03-29T23:38:14.954388+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002460"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970496+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:34:09.970496+0000","last_peered":"2023-03-29T23:34:09.970496+0000","last_clean":"2023-03-29T23:34:09.970496+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:34:09.970496+0000","last_undegraded":"2023-03-29T23:34:09.970496+0000","last_fullsized":"2023-03-29T23:34:09.970496+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:09.970478+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:34:09.970478+0000","last_peered":"2023-03-29T23:34:09.970478+0000","last_clean":"2023-03-29T23:34:09.970478+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:34:09.970478+0000","last_undegraded":"2023-03-29T23:34:09.970478+0000","last_fullsized":"2023-03-29T23:34:09.970478+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":29,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:35:59.979853+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:35:59.979853+0000","last_peered":"2023-03-29T23:35:59.979853+0000","last_clean":"2023-03-29T23:35:59.979853+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:35:59.979853+0000","last_undegraded":"2023-03-29T23:35:59.979853+0000","last_fullsized":"2023-03-29T23:35:59.979853+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:36:03.964375+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:36:03.964375+0000","last_peered":"2023-03-29T23:36:03.964375+0000","last_clean":"2023-03-29T23:36:03.964375+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:36:03.964375+0000","last_undegraded":"2023-03-29T23:36:03.964375+0000","last_fullsized":"2023-03-29T23:36:03.964375+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969945+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:34:04.969945+0000","last_peered":"2023-03-29T23:34:04.969945+0000","last_clean":"2023-03-29T23:34:04.969945+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:34:04.969945+0000","last_undegraded":"2023-03-29T23:34:04.969945+0000","last_fullsized":"2023-03-29T23:34:04.969945+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":28,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:34:04.969967+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:34:04.969967+0000","last_peered":"2023-03-29T23:34:04.969967+0000","last_clean":"2023-03-29T23:34:04.969967+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:34:04.969967+0000","last_undegraded":"2023-03-29T23:34:04.969967+0000","last_fullsized":"2023-03-29T23:34:04.969967+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117068,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:37:09 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27000000000000002},"min":{"1min":0.17599999999999999,"5min":0.16,"15min":0.16},"max":{"1min":0.39300000000000002,"5min":0.51100000000000001,"15min":0.51100000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":0.254,"5min":0.27000000000000002,"15min":0.27400000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":0.41699999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182477,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.28199999999999997,"5min":0.26700000000000002,"15min":0.28499999999999998},"min":{"1min":0.184,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.501},"last":0.20100000000000001},{"interface":"front","average":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.28599999999999998},"min":{"1min":0.19900000000000001,"5min":0.182,"15min":0.182},"max":{"1min":0.43099999999999999,"5min":0.44800000000000001,"15min":0.46000000000000002},"last":0.21299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:38:16.403 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:38:33.493+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:38:33.693+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:38:33.717 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:38:53.348 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:38:53.349 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:38:53.349 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:39:11.577+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:12.196+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:39:12.588+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:39:27.638+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:27.914 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:39:47.470 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:39:47.720+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:39:47.781 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:40:07.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:40:07.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":580,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":10,"modified":"2023-03-29T23:40:00.974542+0000","services":{}},"progress_events":{}} , 2023-03-29T23:40:07.454 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now Ignoring malparsed backtrace: 2023-03-29T23:40:25.145 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:25.459 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:40:41.889+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:40:42.412 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:40:42.735 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:41:04.875+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:04.995 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:41:19.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:41:20.762+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:41:20.923+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:41:38.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:41:38.440 INFO:teuthology.orchestra.run.smithi172.stdout: , 2023-03-29T23:41:38.441 INFO:teuthology.orchestra.run.smithi172.stdout:{"fsid":"285f5dee-5ce0-41f9-a874-4ca884e4570e","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodown,noout,noin flag(s) set","count":17},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 11/200 objects degraded (5.500%), 1 pg degraded","count":1},"muted":false},"POOL_APP_NOT_ENABLED":{"severity":"HEALTH_WARN","summary":{"message":"1 pool(s) do not have an application enabled","count":1},"muted":false}},"mutes":[]},"election_epoch":3,"quorum":[0],"quorum_names":["a"],"quorum_age":671,"monmap":{"epoch":1,"min_mon_release_name":"reef","num_mons":1},"osdmap":{"epoch":37,"num_osds":3,"num_up_osds":3,"osd_up_since":1680132741,"num_in_osds":3,"osd_in_since":1680132724,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":8},{"state_name":"active+recovering+degraded","count":1}],"num_pgs":9,"num_pools":2,"num_objects":100,"data_bytes":47947,"bytes_used":18317312,"bytes_avail":289891975168,"bytes_total":289910292480,"degraded_objects":11,"degraded_total":200,"degraded_ratio":0.055},"fsmap":{"epoch":1,"by_rank":[],"up:standby":0},"mgrmap":{"available":true,"num_standbys":0,"modules":["iostat","nfs","restful"],"services":{}},"servicemap":{"epoch":12,"modified":"2023-03-29T23:41:34.992757+0000","services":{}},"progress_events":{}} Ignoring malparsed backtrace: 2023-03-29T23:41:55.685 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:41:55.698 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":373,"stamp":"2023-03-29T23:41:54.996468+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002193"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993558+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:38:29.993558+0000","last_peered":"2023-03-29T23:38:29.993558+0000","last_clean":"2023-03-29T23:38:29.993558+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:38:29.993558+0000","last_undegraded":"2023-03-29T23:38:29.993558+0000","last_fullsized":"2023-03-29T23:38:29.993558+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:29.993537+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:38:29.993537+0000","last_peered":"2023-03-29T23:38:29.993537+0000","last_clean":"2023-03-29T23:38:29.993537+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:38:29.993537+0000","last_undegraded":"2023-03-29T23:38:29.993537+0000","last_fullsized":"2023-03-29T23:38:29.993537+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:37:59.990713+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:37:59.990713+0000","last_peered":"2023-03-29T23:37:59.990713+0000","last_clean":"2023-03-29T23:37:59.990713+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:37:59.990713+0000","last_undegraded":"2023-03-29T23:37:59.990713+0000","last_fullsized":"2023-03-29T23:37:59.990713+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:08.974789+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:38:08.974789+0000","last_peered":"2023-03-29T23:38:08.974789+0000","last_clean":"2023-03-29T23:38:08.974789+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:38:08.974789+0000","last_undegraded":"2023-03-29T23:38:08.974789+0000","last_fullsized":"2023-03-29T23:38:08.974789+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":38,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992410+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:38:14.992410+0000","last_peered":"2023-03-29T23:38:14.992410+0000","last_clean":"2023-03-29T23:38:14.992410+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:38:14.992410+0000","last_undegraded":"2023-03-29T23:38:14.992410+0000","last_fullsized":"2023-03-29T23:38:14.992410+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":30,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:38:14.992439+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:38:14.992439+0000","last_peered":"2023-03-29T23:38:14.992439+0000","last_clean":"2023-03-29T23:38:14.992439+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:38:14.992439+0000","last_undegraded":"2023-03-29T23:38:14.992439+0000","last_fullsized":"2023-03-29T23:38:14.992439+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117112,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:41:16 2023","interfaces":[{"interface":"back","average":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.16},"max":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.22},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28000000000000003,"15min":0.27700000000000002},"min":{"1min":0.191,"5min":0.19,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.20100000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182523,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:41:17 2023","interfaces":[{"interface":"back","average":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.28000000000000003},"min":{"1min":0.17899999999999999,"5min":0.16700000000000001,"15min":0.16700000000000001},"max":{"1min":0.42899999999999999,"5min":0.47899999999999998,"15min":0.501},"last":0.25},{"interface":"front","average":{"1min":0.30499999999999999,"5min":0.28299999999999997,"15min":0.28399999999999997},"min":{"1min":0.214,"5min":0.182,"15min":0.182},"max":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:42:12.814+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:13.110+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:13.830+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) Ignoring malparsed backtrace: 2023-03-29T23:42:28.932+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:28.996 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:42:48.454+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) , 2023-03-29T23:42:48.799+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.204519+0000 front 2023-03-29T23:32:27.204530+0000 (oldest deadline 2023-03-29T23:32:51.904575+0000) , 2023-03-29T23:42:49.495+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 since back 2023-03-29T23:32:27.176711+0000 front 2023-03-29T23:32:27.176761+0000 (oldest deadline 2023-03-29T23:32:52.476725+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:08.898 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:43:08.912 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-03-29T23:43:07.010413+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002505"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:00.001585+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:40:00.001585+0000","last_peered":"2023-03-29T23:40:00.001585+0000","last_clean":"2023-03-29T23:40:00.001585+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:40:00.001585+0000","last_undegraded":"2023-03-29T23:40:00.001585+0000","last_fullsized":"2023-03-29T23:40:00.001585+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117127,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:42:18 2023","interfaces":[{"interface":"back","average":{"1min":0.26300000000000001,"5min":0.27400000000000002,"15min":0.27000000000000002},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.39500000000000002,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.23799999999999999},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.20100000000000001,"5min":0.191,"15min":0.19},"max":{"1min":0.38100000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.223}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182537,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:42:27 2023","interfaces":[{"interface":"back","average":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27800000000000002},"min":{"1min":0.20699999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.40000000000000002,"5min":0.47899999999999998,"15min":0.501},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.27700000000000002,"5min":0.28100000000000003,"15min":0.28299999999999997},"min":{"1min":0.19,"5min":0.19,"15min":0.182},"max":{"1min":0.432,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.20799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:43:33.852+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:43:34.214+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:34.851+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:43:49.953 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:43:50.220+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:43:50.266 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:07.193 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:44:07.194 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:44:07.235+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:24.547 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":447,"stamp":"2023-03-29T23:44:23.025178+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002346"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005102+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:40:40.005102+0000","last_peered":"2023-03-29T23:40:40.005102+0000","last_clean":"2023-03-29T23:40:40.005102+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:40:40.005102+0000","last_undegraded":"2023-03-29T23:40:40.005102+0000","last_fullsized":"2023-03-29T23:40:40.005102+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:40.005062+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:40:40.005062+0000","last_peered":"2023-03-29T23:40:40.005062+0000","last_clean":"2023-03-29T23:40:40.005062+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:40:40.005062+0000","last_undegraded":"2023-03-29T23:40:40.005062+0000","last_fullsized":"2023-03-29T23:40:40.005062+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:13.985928+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:40:13.985928+0000","last_peered":"2023-03-29T23:40:13.985928+0000","last_clean":"2023-03-29T23:40:13.985928+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:40:13.985928+0000","last_undegraded":"2023-03-29T23:40:13.985928+0000","last_fullsized":"2023-03-29T23:40:13.985928+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":39,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003503+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:40:20.003503+0000","last_peered":"2023-03-29T23:40:20.003503+0000","last_clean":"2023-03-29T23:40:20.003503+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:40:20.003503+0000","last_undegraded":"2023-03-29T23:40:20.003503+0000","last_fullsized":"2023-03-29T23:40:20.003503+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":31,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:40:20.003528+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:40:20.003528+0000","last_peered":"2023-03-29T23:40:20.003528+0000","last_clean":"2023-03-29T23:40:20.003528+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:40:20.003528+0000","last_undegraded":"2023-03-29T23:40:20.003528+0000","last_fullsized":"2023-03-29T23:40:20.003528+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117142,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:43:20 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.28100000000000003,"15min":0.27100000000000002},"min":{"1min":0.20000000000000001,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.40600000000000003,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.29399999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"min":{"1min":0.192,"5min":0.191,"15min":0.19},"max":{"1min":0.41799999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"last":0.23899999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182552,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:43:28 2023","interfaces":[{"interface":"back","average":{"1min":0.28999999999999998,"5min":0.27200000000000002,"15min":0.27800000000000002},"min":{"1min":0.20000000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.443,"5min":0.443,"15min":0.501},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.30599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"min":{"1min":0.20799999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.42899999999999999,"5min":0.46100000000000002,"15min":0.46100000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:44:24.549 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:24.858 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:44:40.724+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:44:41.278+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:44:41.774+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:44:56.919 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:44:57.231 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:45:16.789 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":473,"stamp":"2023-03-29T23:45:15.035469+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002398"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117153,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:44:23 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"min":{"1min":0.222,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.42099999999999999,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.28999999999999998,"5min":0.28799999999999998,"15min":0.27800000000000002},"min":{"1min":0.20499999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.42799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182563,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:44:29 2023","interfaces":[{"interface":"back","average":{"1min":0.253,"5min":0.27100000000000002,"15min":0.27500000000000002},"min":{"1min":0.20599999999999999,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.501},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.27400000000000002,"5min":0.28799999999999998,"15min":0.28299999999999997},"min":{"1min":0.222,"5min":0.19,"15min":0.182},"max":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.35099999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:45:16.790 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:45:17.105 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:45:33.449+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:45:33.732+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:45:34.049 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json Ignoring malparsed backtrace: 2023-03-29T23:45:48.723 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:45:48.736 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":489,"stamp":"2023-03-29T23:45:47.041373+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001983"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:00.012187+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:42:00.012187+0000","last_peered":"2023-03-29T23:42:00.012187+0000","last_clean":"2023-03-29T23:42:00.012187+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:42:00.012187+0000","last_undegraded":"2023-03-29T23:42:00.012187+0000","last_fullsized":"2023-03-29T23:42:00.012187+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117159,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.45900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182569,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.28899999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:46:06.444 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:46:06.444 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:46:06.679+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:46:26.586 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":508,"stamp":"2023-03-29T23:46:25.048633+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002135"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:19.997252+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:42:19.997252+0000","last_peered":"2023-03-29T23:42:19.997252+0000","last_clean":"2023-03-29T23:42:19.997252+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:42:19.997252+0000","last_undegraded":"2023-03-29T23:42:19.997252+0000","last_fullsized":"2023-03-29T23:42:19.997252+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014184+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:42:25.014184+0000","last_peered":"2023-03-29T23:42:25.014184+0000","last_clean":"2023-03-29T23:42:25.014184+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:42:25.014184+0000","last_undegraded":"2023-03-29T23:42:25.014184+0000","last_fullsized":"2023-03-29T23:42:25.014184+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:25.014197+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:42:25.014197+0000","last_peered":"2023-03-29T23:42:25.014197+0000","last_clean":"2023-03-29T23:42:25.014197+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:42:25.014197+0000","last_undegraded":"2023-03-29T23:42:25.014197+0000","last_fullsized":"2023-03-29T23:42:25.014197+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117167,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.28799999999999998,"5min":0.28100000000000003,"15min":0.27300000000000002},"min":{"1min":0.192,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.44400000000000001,"5min":0.46500000000000002,"15min":0.51100000000000001},"last":0.315},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.28799999999999998,"15min":0.28000000000000003},"min":{"1min":0.21199999999999999,"5min":0.191,"15min":0.19},"max":{"1min":0.42799999999999999,"5min":0.44800000000000001,"15min":0.45400000000000001},"last":0.26300000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182577,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:45:29 2023","interfaces":[{"interface":"back","average":{"1min":0.26600000000000001,"5min":0.27200000000000002,"15min":0.27300000000000002},"min":{"1min":0.19500000000000001,"5min":0.17899999999999999,"15min":0.16700000000000001},"max":{"1min":0.42399999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.309},{"interface":"front","average":{"1min":0.25800000000000001,"5min":0.28399999999999997,"15min":0.28100000000000003},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.182},"max":{"1min":0.40699999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:46:26.651+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: 2023-03-29T23:46:46.949 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:46:47.265 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:04.214 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":527,"stamp":"2023-03-29T23:47:03.055977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002350"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":40,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016146+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:42:50.016146+0000","last_peered":"2023-03-29T23:42:50.016146+0000","last_clean":"2023-03-29T23:42:50.016146+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:42:50.016146+0000","last_undegraded":"2023-03-29T23:42:50.016146+0000","last_fullsized":"2023-03-29T23:42:50.016146+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":32,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:42:50.016132+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:42:50.016132+0000","last_peered":"2023-03-29T23:42:50.016132+0000","last_clean":"2023-03-29T23:42:50.016132+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:42:50.016132+0000","last_undegraded":"2023-03-29T23:42:50.016132+0000","last_fullsized":"2023-03-29T23:42:50.016132+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117174,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.26000000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182584,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.193},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.216}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:04.335+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:04.729+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) Ignoring malparsed backtrace: dumped all , 2023-03-29T23:47:24.092 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":537,"stamp":"2023-03-29T23:47:23.060043+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002270"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027591+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:45:00.027591+0000","last_peered":"2023-03-29T23:45:00.027591+0000","last_clean":"2023-03-29T23:45:00.027591+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:45:00.027591+0000","last_undegraded":"2023-03-29T23:45:00.027591+0000","last_fullsized":"2023-03-29T23:45:00.027591+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:45:00.027575+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:45:00.027575+0000","last_peered":"2023-03-29T23:45:00.027575+0000","last_clean":"2023-03-29T23:45:00.027575+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:45:00.027575+0000","last_undegraded":"2023-03-29T23:45:00.027575+0000","last_fullsized":"2023-03-29T23:45:00.027575+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:00.022015+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:44:00.022015+0000","last_peered":"2023-03-29T23:44:00.022015+0000","last_clean":"2023-03-29T23:44:00.022015+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:44:00.022015+0000","last_undegraded":"2023-03-29T23:44:00.022015+0000","last_fullsized":"2023-03-29T23:44:00.022015+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:25.006764+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:44:25.006764+0000","last_peered":"2023-03-29T23:44:25.006764+0000","last_clean":"2023-03-29T23:44:25.006764+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:44:25.006764+0000","last_undegraded":"2023-03-29T23:44:25.006764+0000","last_fullsized":"2023-03-29T23:44:25.006764+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":41,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024499+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:44:30.024499+0000","last_peered":"2023-03-29T23:44:30.024499+0000","last_clean":"2023-03-29T23:44:30.024499+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:44:30.024499+0000","last_undegraded":"2023-03-29T23:44:30.024499+0000","last_fullsized":"2023-03-29T23:44:30.024499+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":33,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:44:30.024512+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:44:30.024512+0000","last_peered":"2023-03-29T23:44:30.024512+0000","last_clean":"2023-03-29T23:44:30.024512+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:44:30.024512+0000","last_undegraded":"2023-03-29T23:44:30.024512+0000","last_fullsized":"2023-03-29T23:44:30.024512+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117178,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27300000000000002},"min":{"1min":0.20399999999999999,"5min":0.16900000000000001,"15min":0.16},"max":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.27200000000000002,"5min":0.28399999999999997,"15min":0.28000000000000003},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182588,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:46:29 2023","interfaces":[{"interface":"back","average":{"1min":0.28000000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"min":{"1min":0.20699999999999999,"5min":0.19500000000000001,"15min":0.16700000000000001},"max":{"1min":0.36499999999999999,"5min":0.45200000000000001,"15min":0.501},"last":0.224},{"interface":"front","average":{"1min":0.28000000000000003,"5min":0.27900000000000003,"15min":0.28000000000000003},"min":{"1min":0.22900000000000001,"5min":0.19,"15min":0.182},"max":{"1min":0.36899999999999999,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.23499999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:47:24.094 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:47:41.245+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:41.405 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:47:56.255+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:47:56.667+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:47:57.281+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:12.986 INFO:tasks.ceph.ceph_manager.ceph:no progress seen, keeping timeout for now , 2023-03-29T23:48:12.987 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:48:13.224+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:34.315+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:48:34.761+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:35.329+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:48:52.495 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json , 2023-03-29T23:48:52.778+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:48:52.884 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: 2023-03-29T23:49:12.131+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:49:12.427 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:49:12.752 INFO:teuthology.orchestra.run.smithi172.stdout: Ignoring malparsed backtrace: dumped all , 2023-03-29T23:49:32.345 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-03-29T23:49:31.084790+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001969"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117204,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.29199999999999998,"15min":0.27800000000000002},"min":{"1min":0.23400000000000001,"5min":0.191,"15min":0.16900000000000001},"max":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.217},{"interface":"front","average":{"1min":0.318,"5min":0.29499999999999998,"15min":0.28399999999999997},"min":{"1min":0.24299999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.192}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182614,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:48:35 2023","interfaces":[{"interface":"back","average":{"1min":0.248,"5min":0.26200000000000001,"15min":0.26700000000000002},"min":{"1min":0.184,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.36599999999999999,"5min":0.45200000000000001,"15min":0.47899999999999998},"last":0.16900000000000001},{"interface":"front","average":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.27700000000000002},"min":{"1min":0.19700000000000001,"5min":0.17799999999999999,"15min":0.17799999999999999},"max":{"1min":0.38200000000000001,"5min":0.46999999999999997,"15min":0.46999999999999997},"last":0.19500000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:49:32.346 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean Ignoring malparsed backtrace: 2023-03-29T23:49:49.639 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:49:49.652 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-03-29T23:49:49.088127+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002184"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:00.032892+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:46:00.032892+0000","last_peered":"2023-03-29T23:46:00.032892+0000","last_clean":"2023-03-29T23:46:00.032892+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:46:00.032892+0000","last_undegraded":"2023-03-29T23:46:00.032892+0000","last_fullsized":"2023-03-29T23:46:00.032892+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:30.017023+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:46:30.017023+0000","last_peered":"2023-03-29T23:46:30.017023+0000","last_clean":"2023-03-29T23:46:30.017023+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:46:30.017023+0000","last_undegraded":"2023-03-29T23:46:30.017023+0000","last_fullsized":"2023-03-29T23:46:30.017023+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117207,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:49:35 2023","interfaces":[{"interface":"back","average":{"1min":0.26800000000000002,"5min":0.28899999999999998,"15min":0.27600000000000002},"min":{"1min":0.19,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.36799999999999999,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.223},{"interface":"front","average":{"1min":0.26600000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"min":{"1min":0.192,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.36899999999999999,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.24399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182617,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:49:36 2023","interfaces":[{"interface":"back","average":{"1min":0.25600000000000001,"5min":0.26200000000000001,"15min":0.26600000000000001},"min":{"1min":0.14899999999999999,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.223},{"interface":"front","average":{"1min":0.26700000000000002,"5min":0.26800000000000002,"15min":0.27600000000000002},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.25800000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:04.992+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:50:05.187+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:50:06.017+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:50:24.066 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:50:24.067 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:50:24.067 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:50:42.198 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all , 2023-03-29T23:50:42.212 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":636,"stamp":"2023-03-29T23:50:41.098507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002306"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036254+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:46:35.036254+0000","last_peered":"2023-03-29T23:46:35.036254+0000","last_clean":"2023-03-29T23:46:35.036254+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:46:35.036254+0000","last_undegraded":"2023-03-29T23:46:35.036254+0000","last_fullsized":"2023-03-29T23:46:35.036254+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:46:35.036274+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:46:35.036274+0000","last_peered":"2023-03-29T23:46:35.036274+0000","last_clean":"2023-03-29T23:46:35.036274+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:46:35.036274+0000","last_undegraded":"2023-03-29T23:46:35.036274+0000","last_fullsized":"2023-03-29T23:46:35.036274+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117218,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.255},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.27400000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182628,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.247}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} Ignoring malparsed backtrace: 2023-03-29T23:50:59.199 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:50:59.512 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: dumped all , 2023-03-29T23:51:21.865 INFO:teuthology.orchestra.run.smithi172.stdout:{"pg_ready":true,"pg_map":{"version":656,"stamp":"2023-03-29T23:51:21.106020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":19,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":17888,"kb_used_data":1672,"kb_used_omap":0,"kb_used_meta":16192,"kb_avail":283097632,"statfs":{"total":289910292480,"available":289891975168,"internally_reserved":0,"allocated":1712128,"data_stored":376820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":16580608},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002029"},"pg_stats":[{"pgid":"1.7","version":"0'0","reported_seq":42,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039789+0000","last_change":"2023-03-29T23:32:23.960281+0000","last_active":"2023-03-29T23:47:10.039789+0000","last_peered":"2023-03-29T23:47:10.039789+0000","last_clean":"2023-03-29T23:47:10.039789+0000","last_became_active":"2023-03-29T23:32:23.960003+0000","last_became_peered":"2023-03-29T23:32:23.960003+0000","last_unstale":"2023-03-29T23:47:10.039789+0000","last_undegraded":"2023-03-29T23:47:10.039789+0000","last_fullsized":"2023-03-29T23:47:10.039789+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:41:04.643152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"0'0","reported_seq":34,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:47:10.039763+0000","last_change":"2023-03-29T23:32:06.972950+0000","last_active":"2023-03-29T23:47:10.039763+0000","last_peered":"2023-03-29T23:47:10.039763+0000","last_clean":"2023-03-29T23:47:10.039763+0000","last_became_active":"2023-03-29T23:32:06.972693+0000","last_became_peered":"2023-03-29T23:32:06.972693+0000","last_unstale":"2023-03-29T23:47:10.039763+0000","last_undegraded":"2023-03-29T23:47:10.039763+0000","last_fullsized":"2023-03-29T23:47:10.039763+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T08:22:15.848491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965240+0000","last_change":"2023-03-29T23:32:22.962678+0000","last_active":"2023-03-29T23:32:22.965240+0000","last_peered":"2023-03-29T23:32:22.965240+0000","last_clean":"2023-03-29T23:32:22.965240+0000","last_became_active":"2023-03-29T23:32:22.962404+0000","last_became_peered":"2023-03-29T23:32:22.962404+0000","last_unstale":"2023-03-29T23:32:22.965240+0000","last_undegraded":"2023-03-29T23:32:22.965240+0000","last_fullsized":"2023-03-29T23:32:22.965240+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:55:10.773807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:00.043865+0000","last_change":"2023-03-29T23:32:06.970156+0000","last_active":"2023-03-29T23:48:00.043865+0000","last_peered":"2023-03-29T23:48:00.043865+0000","last_clean":"2023-03-29T23:48:00.043865+0000","last_became_active":"2023-03-29T23:32:06.969889+0000","last_became_peered":"2023-03-29T23:32:06.969889+0000","last_unstale":"2023-03-29T23:48:00.043865+0000","last_undegraded":"2023-03-29T23:48:00.043865+0000","last_fullsized":"2023-03-29T23:48:00.043865+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T00:08:16.832113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"0'0","reported_seq":36,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:35.027177+0000","last_change":"2023-03-29T23:32:05.970413+0000","last_active":"2023-03-29T23:48:35.027177+0000","last_peered":"2023-03-29T23:48:35.027177+0000","last_clean":"2023-03-29T23:48:35.027177+0000","last_became_active":"2023-03-29T23:32:05.970093+0000","last_became_peered":"2023-03-29T23:32:05.970093+0000","last_unstale":"2023-03-29T23:48:35.027177+0000","last_undegraded":"2023-03-29T23:48:35.027177+0000","last_fullsized":"2023-03-29T23:48:35.027177+0000","mapping_epoch":29,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":30,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:21:24.126541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"30'101","reported_seq":132,"reported_epoch":36,"state":"active+recovering+degraded","last_fresh":"2023-03-29T23:32:22.965531+0000","last_change":"2023-03-29T23:32:22.965531+0000","last_active":"2023-03-29T23:32:22.965531+0000","last_peered":"2023-03-29T23:32:22.965531+0000","last_clean":"2023-03-29T23:31:30.451497+0000","last_became_active":"2023-03-29T23:32:22.964517+0000","last_became_peered":"2023-03-29T23:32:22.964517+0000","last_unstale":"2023-03-29T23:32:22.965531+0000","last_undegraded":"2023-03-29T23:32:22.958773+0000","last_fullsized":"2023-03-29T23:32:22.965531+0000","mapping_epoch":35,"log_start":"18'95","ondisk_log_start":"18'95","created":16,"last_epoch_clean":17,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:52.905334+0000","last_clean_scrub_stamp":"2023-03-29T23:30:52.905334+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":95,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1","objects":11},{"shards":"1,2","objects":89}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"0'0","reported_seq":43,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047612+0000","last_change":"2023-03-29T23:32:23.963311+0000","last_active":"2023-03-29T23:48:40.047612+0000","last_peered":"2023-03-29T23:48:40.047612+0000","last_clean":"2023-03-29T23:48:40.047612+0000","last_became_active":"2023-03-29T23:32:23.963022+0000","last_became_peered":"2023-03-29T23:32:23.963022+0000","last_unstale":"2023-03-29T23:48:40.047612+0000","last_undegraded":"2023-03-29T23:48:40.047612+0000","last_fullsized":"2023-03-29T23:48:40.047612+0000","mapping_epoch":36,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":37,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T10:53:54.599224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"0'0","reported_seq":35,"reported_epoch":37,"state":"active+clean","last_fresh":"2023-03-29T23:48:40.047626+0000","last_change":"2023-03-29T23:32:06.973000+0000","last_active":"2023-03-29T23:48:40.047626+0000","last_peered":"2023-03-29T23:48:40.047626+0000","last_clean":"2023-03-29T23:48:40.047626+0000","last_became_active":"2023-03-29T23:32:06.972749+0000","last_became_peered":"2023-03-29T23:32:06.972749+0000","last_unstale":"2023-03-29T23:48:40.047626+0000","last_undegraded":"2023-03-29T23:48:40.047626+0000","last_fullsized":"2023-03-29T23:48:40.047626+0000","mapping_epoch":30,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":31,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T06:31:47.306268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.5","version":"0'0","reported_seq":22,"reported_epoch":36,"state":"active+clean","last_fresh":"2023-03-29T23:32:22.965211+0000","last_change":"2023-03-29T23:32:22.960130+0000","last_active":"2023-03-29T23:32:22.965211+0000","last_peered":"2023-03-29T23:32:22.965211+0000","last_clean":"2023-03-29T23:32:22.965211+0000","last_became_active":"2023-03-29T23:32:22.959865+0000","last_became_peered":"2023-03-29T23:32:22.959865+0000","last_unstale":"2023-03-29T23:32:22.965211+0000","last_undegraded":"2023-03-29T23:32:22.965211+0000","last_fullsized":"2023-03-29T23:32:22.965211+0000","mapping_epoch":35,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":36,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-03-29T23:30:38.809679+0000","last_clean_scrub_stamp":"2023-03-29T23:30:38.809679+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-03-31T01:41:55.409164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":1,"stat_sum":{"num_bytes":47947,"num_objects":100,"num_object_clones":0,"num_object_copies":200,"num_objects_missing_on_primary":11,"num_objects_missing":11,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":101,"num_write_kb":101,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":782336,"data_stored":91900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6,"ondisk_log_size":6,"up":2,"acting":2,"num_store_stats":3},{"poolid":1,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":16,"acting":16,"num_store_stats":3}],"osd_stats":[{"osd":2,"up_from":35,"seq":150323855363,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6048,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365792,"statfs":{"total":96636764160,"available":96630571008,"internally_reserved":0,"allocated":679936,"data_stored":138544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34499999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29199999999999998}]}]},{"osd":1,"up_from":27,"seq":115964117226,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":6088,"kb_used_data":704,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":94365752,"statfs":{"total":96636764160,"available":96630530048,"internally_reserved":0,"allocated":720896,"data_stored":143202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Wed Mar 29 23:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.26000000000000001,"5min":0.28299999999999997,"15min":0.27600000000000002},"min":{"1min":0.19700000000000001,"5min":0.19,"15min":0.16900000000000001},"max":{"1min":0.372,"5min":0.48899999999999999,"15min":0.51100000000000001},"last":0.21199999999999999},{"interface":"front","average":{"1min":0.26400000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"min":{"1min":0.184,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":0.39400000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"last":0.23799999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28399999999999997}]}]},{"osd":0,"up_from":25,"seq":107374182636,"num_pgs":7,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":5752,"kb_used_data":304,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":94366088,"statfs":{"total":96636764160,"available":96630874112,"internally_reserved":0,"allocated":311296,"data_stored":95074,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Wed Mar 29 23:50:37 2023","interfaces":[{"interface":"back","average":{"1min":0.23999999999999999,"5min":0.25700000000000001,"15min":0.26500000000000001},"min":{"1min":0.19500000000000001,"5min":0.14199999999999999,"15min":0.14199999999999999},"max":{"1min":0.38,"5min":0.43099999999999999,"15min":0.47899999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.248,"5min":0.26600000000000001,"15min":0.27500000000000002},"min":{"1min":0.17699999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":0.40200000000000002,"5min":0.46600000000000003,"15min":0.46999999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":413696,"data_stored":48430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":368640,"data_stored":43470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} , 2023-03-29T23:51:21.939+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:51:44.127 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json , 2023-03-29T23:51:44.452 INFO:teuthology.orchestra.run.smithi172.stdout: , dumped all Ignoring malparsed backtrace: 2023-03-29T23:51:58.836+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) , 2023-03-29T23:51:59.704+0000 7f33247b0700 -1 osd.0 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.758231+0000 (oldest deadline 2023-03-29T23:43:20.758231+0000) , 2023-03-29T23:51:59.840+0000 7f01f5b44700 -1 osd.1 37 heartbeat_check: no reply from 172.21.15.172:6810 osd.2 ever on either front or back, first ping sent 2023-03-29T23:43:00.830732+0000 (oldest deadline 2023-03-29T23:43:20.830732+0000) Ignoring malparsed backtrace: 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:PG 2.0 is not active+clean , 2023-03-29T23:52:18.690 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:18.691 DEBUG:teuthology.orchestra.run.smithi172:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph status --format=json Ignoring malparsed backtrace: 2023-03-29T23:52:35.787 INFO:tasks.ceph.ceph_manager.ceph:{'pgid': '2.0', 'version': "30'101", 'reported_seq': 132, 'reported_epoch': 36, 'state': 'active+recovering+degraded', 'last_fresh': '2023-03-29T23:32:22.965531+0000', 'last_change': '2023-03-29T23:32:22.965531+0000', 'last_active': '2023-03-29T23:32:22.965531+0000', 'last_peered': '2023-03-29T23:32:22.965531+0000', 'last_clean': '2023-03-29T23:31:30.451497+0000', 'last_became_active': '2023-03-29T23:32:22.964517+0000', 'last_became_peered': '2023-03-29T23:32:22.964517+0000', 'last_unstale': '2023-03-29T23:32:22.965531+0000', 'last_undegraded': '2023-03-29T23:32:22.958773+0000', 'last_fullsized': '2023-03-29T23:32:22.965531+0000', 'mapping_epoch': 35, 'log_start': "18'95", 'ondisk_log_start': "18'95", 'created': 16, 'last_epoch_clean': 17, 'parent': '0.0', 'parent_split_bits': 0, 'last_scrub': "0'0", 'last_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_deep_scrub': "0'0", 'last_deep_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'last_clean_scrub_stamp': '2023-03-29T23:30:52.905334+0000', 'objects_scrubbed': 0, 'log_size': 6, 'log_dups_size': 95, 'ondisk_log_size': 6, 'stats_invalid': False, 'dirty_stats_invalid': False, 'omap_stats_invalid': False, 'hitset_stats_invalid': False, 'hitset_bytes_stats_invalid': False, 'pin_stats_invalid': False, 'manifest_stats_invalid': False, 'snaptrimq_len': 0, 'last_scrub_duration': 0, 'scrub_schedule': 'periodic scrub scheduled @ 2023-03-31T03:26:10.130921+0000', 'scrub_duration': 0, 'objects_trimmed': 0, 'snaptrim_duration': 0, 'stat_sum': {'num_bytes': 47947, 'num_objects': 100, 'num_object_clones': 0, 'num_object_copies': 200, 'num_objects_missing_on_primary': 11, 'num_objects_missing': 11, 'num_objects_degraded': 11, 'num_objects_misplaced': 0, 'num_objects_unfound': 0, 'num_objects_dirty': 100, 'num_whiteouts': 0, 'num_read': 0, 'num_read_kb': 0, 'num_write': 101, 'num_write_kb': 101, 'num_scrub_errors': 0, 'num_shallow_scrub_errors': 0, 'num_deep_scrub_errors': 0, 'num_objects_recovered': 0, 'num_bytes_recovered': 0, 'num_keys_recovered': 0, 'num_objects_omap': 0, 'num_objects_hit_set_archive': 0, 'num_bytes_hit_set_archive': 0, 'num_flush': 0, 'num_flush_kb': 0, 'num_evict': 0, 'num_evict_kb': 0, 'num_promote': 0, 'num_flush_mode_high': 0, 'num_flush_mode_low': 0, 'num_evict_mode_some': 0, 'num_evict_mode_full': 0, 'num_objects_pinned': 0, 'num_legacy_snapsets': 0, 'num_large_omap_objects': 0, 'num_objects_manifest': 0, 'num_omap_bytes': 0, 'num_omap_keys': 0, 'num_objects_repaired': 0}, 'up': [2, 1], 'acting': [2, 1], 'avail_no_missing': ['1'], 'object_location_counts': [{'shards': '1', 'objects': 11}, {'shards': '1,2', 'objects': 89}], 'blocked_by': [], 'up_primary': 2, 'acting_primary': 2, 'purged_snaps': []} , 2023-03-29T23:52:35.787 ERROR:teuthology.contextutil:Saw exception from nested tasks , Traceback (most recent call last): Ignoring malparsed backtrace: 2023-03-29T23:52:40.177 DEBUG:teuthology.run_tasks:Unwinding manager pcp , 2023-03-29T23:52:40.191 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer , 2023-03-29T23:52:40.201 INFO:teuthology.task.internal:Duration was 1969.744679 seconds Misunderstood line: Found 23 distinct failure reasons Timeout 6h running clone.client.0/qa/workunits/rados/test.sh 3 jobs: ['7224788', '7224248', '7224628'] suites intersection: ['ceph', 'clusters/{fixed-2', 'msgr-failures/few', 'msgr/async-v1only', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_api_tests'] suites union: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'd-thrash/none', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v1only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_api_tests', 'thrashosds-health}', 'validater/lockdep}', 'validater/valgrind}'] Failure: Command crashed: 'sudo ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-0 --journal-path /var/lib/ceph/osd/ceph-0/journal --force --op remove --pgid 3.b' 1 jobs: ['7224547'] suites: ['rados/objectstore/{backends/ceph_objectstore_tool', 'supported-random-distro$/{ubuntu_latest}}'] Dead: b'2023-03-30T10:42:46.082 DEBUG:teuthology.exit:Finished running handlers' 62 jobs suites intersection: [] suites union: ['0-size-min-size-overrides/2-size-2-min-size', '0-size-min-size-overrides/3-size-2-min-size', '0-start', '1-install/nautilus', '1-install/nautilus-v2only', '1-install/octopus', '1-install/quincy', '1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '1-tasks', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-active-recovery}', '2-recovery-overrides/{more-async-partial-recovery}', '2-recovery-overrides/{more-async-recovery}', '2-recovery-overrides/{more-partial-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'backoff/peering', 'backoff/peering_and_degraded', 'ceph', 'cluster/{12-osds', 'clusters/{fixed-2', 'clusters/{fixed-4', 'clusters/{openstack', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'fast/fast', 'fast/normal', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/osd-delay', 'msgr-failures/osd-dispatch-delay', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/singleton/{all/thrash-backfill-full', 'rados/thrash-erasure-code-big/{ceph', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash-erasure-code/{ceph', 'rados/thrash-old-clients/{0-distro$/{centos_8.stream_container_tools}', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/upgrade/parallel/{0-random-distro$/{ubuntu_20.04}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'recovery-overrides/{default}', 'recovery-overrides/{more-active-recovery}', 'recovery-overrides/{more-async-partial-recovery}', 'recovery-overrides/{more-async-recovery}', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'supported-random-distro$/{ubuntu_latest}', 'test_rbd_api', 'test_rbd_python}}', 'thrashers/careful', 'thrashers/default', 'thrashers/fastread', 'thrashers/mapgap', 'thrashers/minsize_recovery', 'thrashers/morepggrow', 'thrashers/none', 'thrashers/pggrow', 'thrashosds-health', 'three-plus-one}', 'upgrade-sequence', 'workload/{ec-rados-default', 'workloads/cache-agent-small}', 'workloads/cache-pool-snaps-readproxy}', 'workloads/cache-pool-snaps}', 'workloads/cache-snaps-balanced}', 'workloads/cache-snaps}', 'workloads/dedup-io-snaps}', 'workloads/ec-pool-snaps-few-objects-overwrites}', 'workloads/ec-rados-plugin=clay-k=4-m=2}', 'workloads/ec-rados-plugin=isa-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=2-m=1}', 'workloads/ec-rados-plugin=jerasure-k=3-m=1}', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}', 'workloads/ec-rados-plugin=lrc-k=4-m=2-l=3}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}', 'workloads/ec-small-objects-balanced}', 'workloads/ec-small-objects-fast-read-overwrites}', 'workloads/ec-small-objects-fast-read}', 'workloads/ec-small-objects-many-deletes}', 'workloads/ec-small-objects-overwrites}', 'workloads/ec-small-objects}', 'workloads/ec-snaps-few-objects-overwrites}', 'workloads/pool-snaps-few-objects}', 'workloads/radosbench}', 'workloads/redirect_promote_tests}', 'workloads/redirect_set_object}', 'workloads/set-chunks-read}', 'workloads/small-objects-balanced}', 'workloads/small-objects-localized}', 'workloads/small-objects}', 'workloads/snaps-few-objects-balanced}', 'workloads/snaps-few-objects-localized}', 'workloads/snaps-few-objects}', 'workloads/write_fadvise_dontneed}'] Dead: b'OSError: Socket is closed' 2 jobs: ['7224866', '7224868'] suites intersection: ['openstack}', 'rados', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashosds-health'] suites union: ['ceph', 'cluster/{12-osds', 'clusters/{fixed-2', 'fast/normal', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/osd-dispatch-delay', 'objectstore/bluestore-low-osd-mem-target', 'openstack}', 'rados', 'rados/thrash-erasure-code-big/{ceph', 'rados/thrash-erasure-code-overwrites/{bluestore-bitmap', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/fastread', 'thrashers/mapgap', 'thrashosds-health', 'workloads/ec-rados-plugin=jerasure-k=4-m=2}', 'workloads/ec-snaps-few-objects-overwrites}'] Failure: Command failed on smithi019 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull' 4 jobs: ['7224539', '7224739', '7224811', '7224259'] suites intersection: ['0-kubeadm', '0-nvme-loop', '1-rook', 'k8s/1.21', 'rados/rook/smoke/{0-distro/ubuntu_20.04'] suites union: ['0-kubeadm', '0-nvme-loop', '1-rook', '2-workload/none', '2-workload/radosbench', 'cluster/1-node', 'cluster/3-node', 'k8s/1.21', 'net/calico', 'net/flannel', 'net/host', 'rados/rook/smoke/{0-distro/ubuntu_20.04', 'rook/1.7.2}', 'rook/master}'] Failure: reached maximum tries (3650) after waiting for 21900 seconds 3 jobs: ['7224453', '7224774', '7224792'] suites intersection: ['mon_election/connectivity', 'rados'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-3}', 'backoff/normal', 'ceph', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'fast/normal', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zstd', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/singleton/{all/thrash-eio', 'rados/thrash-erasure-code/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'recovery-overrides/{more-partial-recovery}', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{ubuntu_latest}', 'supported-random-distro$/{ubuntu_latest}}', 'thrashers/default', 'thrashers/morepggrow', 'thrashosds-health', 'workloads/admin_socket_objecter_requests}', 'workloads/ec-radosbench}'] Timeout 3h running clone.client.0/qa/workunits/rados/test.sh 4 jobs: ['7224469', '7224401', '7224821', '7224366'] suites intersection: ['mon_election/connectivity', 'rados'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-partial-recovery}', '3-scrub-overrides/{default}', 'backoff/peering', 'ceph', 'clusters/9-mons', 'clusters/{fixed-2', 'crc-failures/default', 'd-balancer/on', 'mon_election/connectivity', 'msgr-failures/few', 'msgr-failures/many', 'msgr-failures/osd-dispatch-delay', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/basic/{ceph', 'rados/monthrash/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'tasks/rados_api_tests}', 'thrashers/default', 'thrashers/sync', 'thrashers/sync-many', 'thrashosds-health', 'workloads/rados_api_tests}'] Timeout 3h running clone.client.0/qa/workunits/cls/test_cls_2pc_queue.sh 1 jobs: ['7224374'] suites: ['ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async-v2only', 'objectstore/bluestore-comp-zlib', 'openstack}', 'rados', 'rados/verify/{centos_latest', 'tasks/rados_cls_all', 'thrashosds-health}', 'validater/valgrind}'] Valgrind: osd (Leak_PossiblyLost) 2 jobs: ['7224755', '7224767'] suites intersection: [] suites union: ['2-inject-leak/none', 'centos_latest}', 'ceph', 'clusters/{fixed-2', 'd-thrash/default/{default', 'mon_election/connectivity', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/valgrind-leaks/{1-start', 'rados/verify/{centos_latest', 'tasks/mon_recovery', 'thrashosds-health}', 'validater/valgrind}'] Timeout 120 running ceph --cluster ceph pg dump --format=json 2 jobs: ['7224785', '7224861'] suites intersection: ['openstack}', 'rados', 'thrashosds-health'] suites union: ['1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'clusters/{fixed-4', 'crc-failures/default', 'd-balancer/on', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr/async', 'objectstore/bluestore-low-osd-mem-target', 'objectstore/bluestore-stupid', 'openstack}', 'rados', 'rados/thrash-erasure-code-shec/{ceph', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'recovery-overrides/{more-active-recovery}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{ubuntu_latest}', 'thrashers/careful', 'thrashers/pggrow', 'thrashosds-health', 'workloads/dedup-io-mixed}', 'workloads/ec-rados-plugin=shec-k=4-m=3-c=2}'] Failure: Test failure: test_update_export_with_invalid_values (tasks.cephfs.test_nfs.TestNFS) 1 jobs: ['7224836'] suites: ['agent/on', 'mon_election/classic', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'task/test_nfs}'] Failure: Command failed (workunit test rados/test_envlibrados_for_rocksdb.sh) on smithi033 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e369dd579b62fff69e4b88c7d4fb7419fe60653c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rados/test_envlibrados_for_rocksdb.sh' 2 jobs: ['7224585', '7224404'] suites intersection: ['mon_election/connectivity', 'msgr-failures/none', 'msgr/async-v2only', 'rados', 'test_envlibrados_for_rocksdb}'] suites union: ['mon_election/connectivity', 'msgr-failures/none', 'msgr/async-v2only', 'objectstore/bluestore-bitmap', 'objectstore/bluestore-comp-snappy', 'rados', 'rados/singleton/{all/test_envlibrados_for_rocksdb/{supported/centos_latest', 'rados/singleton/{all/test_envlibrados_for_rocksdb/{supported/rhel_latest', 'supported-random-distro$/{centos_8}}', 'supported-random-distro$/{ubuntu_latest}}', 'test_envlibrados_for_rocksdb}'] Failure: Command failed (workunit test scrub/osd-mapper.sh) on smithi084 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e369dd579b62fff69e4b88c7d4fb7419fe60653c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/scrub/osd-mapper.sh' 1 jobs: ['7224423'] suites: ['rados/standalone/{supported-random-distro$/{centos_8}', 'workloads/scrub}'] Failure: Command failed (workunit test osd-backfill/osd-backfill-space.sh) on smithi121 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e369dd579b62fff69e4b88c7d4fb7419fe60653c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd-backfill/osd-backfill-space.sh' 1 jobs: ['7224864'] suites: ['rados/standalone/{supported-random-distro$/{rhel_8}', 'workloads/osd-backfill}'] Crash: reached maximum tries (500) after waiting for 3000 seconds ceph version 18.0.0-2999-ge369dd57 (e369dd579b62fff69e4b88c7d4fb7419fe60653c) reef (dev) 1: /lib64/libpthread.so.0(+0x12ce0) [0x7f658ed4ace0] 2: gsignal() 3: abort() 4: (ceph::__ceph_abort(char const*, int, char const*, std::__cxx11::basic_string, std::allocator > const&)+0x197) [0x7f659008f4df] 5: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, ceph::os::Transaction*)+0xc39) [0x55f38e6697e9] 6: (BlueStore::queue_transactions(boost::intrusive_ptr&, std::vector >&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0x2cf) [0x55f38e67e86f] 7: (ObjectStore::queue_transaction(boost::intrusive_ptr&, ceph::os::Transaction&&, boost::intrusive_ptr, ThreadPool::TPHandle*)+0xae) [0x55f38e08205e] 8: (OSD::recursive_remove_collection(ceph::common::CephContext*, ObjectStore*, spg_t, coll_t, pool_pg_num_history_t&, GlobalSnapMapper*)+0x2fa) [0x55f38e0c3c2a] 9: ceph-objectstore-tool(+0x4e684a) [0x55f38e06d84a] 10: ceph-objectstore-tool(+0x4e6c10) [0x55f38e06dc10] 11: main() 12: __libc_start_main() 13: _start() 5 jobs: ['7224466', '7224474', '7224304', '7224442', '7224822'] suites intersection: [] suites union: ['1-pg-log-overrides/normal_pg_log', '1-pg-log-overrides/short_pg_log', '2-recovery-overrides/{default}', '2-recovery-overrides/{more-async-recovery}', '3-scrub-overrides/{default}', '3-scrub-overrides/{max-simultaneous-scrubs-2}', 'backoff/normal', 'backoff/peering_and_degraded', 'ceph', 'clusters/{fixed-2', 'crc-failures/bad_map_crc_failure', 'crc-failures/default', 'd-balancer/crush-compat', 'd-balancer/on', 'd-thrash/default/{default', 'mon_election/classic', 'mon_election/connectivity', 'msgr-failures/fastclose', 'msgr-failures/few', 'msgr-failures/none', 'msgr/async', 'msgr/async-v1only', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'objectstore/bluestore-comp-snappy', 'objectstore/bluestore-comp-zlib', 'objectstore/bluestore-hybrid', 'openstack}', 'rados', 'rados/singleton/{all/thrash-rados/{thrash-rados', 'rados/standalone/{supported-random-distro$/{ubuntu_latest}', 'rados/thrash/{0-size-min-size-overrides/2-size-2-min-size', 'rados/thrash/{0-size-min-size-overrides/3-size-2-min-size', 'rados/verify/{centos_latest', 'supported-random-distro$/{centos_8}', 'supported-random-distro$/{rhel_8}', 'supported-random-distro$/{rhel_8}}', 'tasks/rados_cls_all', 'thrashers/mapgap', 'thrashers/morepggrow', 'thrashosds-health', 'thrashosds-health}', 'validater/valgrind}', 'workloads/osd}', 'workloads/radosbench-high-concurrency}', 'workloads/radosbench}'] Failure: Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi187 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=e369dd579b62fff69e4b88c7d4fb7419fe60653c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh' 2 jobs: ['7224740', '7224274'] suites intersection: ['debug/mgr', 'rados/dashboard/{0-single-container-host', 'tasks/e2e}'] suites union: ['debug/mgr', 'mon_election/classic', 'mon_election/connectivity', 'rados/dashboard/{0-single-container-host', 'random-objectstore$/{bluestore-comp-lz4}', 'random-objectstore$/{bluestore-low-osd-mem-target}', 'tasks/e2e}'] Dead: None 1 jobs: ['7224388'] Failure: Test failure: test_cephfs_mirror (tasks.cephadm_cases.test_cli.TestCephadmCLI) 1 jobs: ['7224720'] suites: ['agent/on', 'mon_election/classic', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools', 'task/test_orch_cli}'] Failure: SELinux denials found on ubuntu@smithi129.front.sepia.ceph.com: ['type=AVC msg=audit(1680086121.146:19741): avc: denied { ioctl } for pid=122930 comm="iptables" path="/var/lib/containers/storage/overlay/6ff038bc32b14e02d416faa58673f995b5401f6d8a39eb39a96f02332077c0ff/merged" dev="overlay" ino=3409167 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1'] 1 jobs: ['7224499'] suites: ['agent/on', 'mon_election/classic', 'rados/cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun', 'task/test_cephadm}'] Timeout 3h running clone.client.0/qa/workunits/rbd/test_librbd.sh 1 jobs: ['7224561'] suites: ['0-start', '1-tasks', 'mon_election/classic', 'rados/upgrade/parallel/{0-random-distro$/{centos_8.stream_container_tools_crun}', 'rados_api', 'rados_loadgenbig', 'rbd_import_export', 'test_rbd_api', 'test_rbd_python}}', 'upgrade-sequence', 'workload/{ec-rados-default'] Failure: Command crashed: 'sudo adjust-ulimits ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-2 --journal-path /var/lib/ceph/osd/ceph-2/journal --log-file=/var/log/ceph/objectstore_tool.$$.log --op export-remove --pgid 2.0 --file /home/ubuntu/cephtest/exp.32116.out' 1 jobs: ['7224669'] suites: ['mon_election/classic', 'msgr-failures/many', 'msgr/async-v1only', 'objectstore/bluestore-hybrid', 'rados', 'rados/singleton/{all/divergent_priors2', 'supported-random-distro$/{centos_8}}'] Failure: failed to complete snap trimming before timeout 1 jobs: ['7224701'] suites: ['ceph', 'clusters/{fixed-2', 'mon_election/classic', 'msgr-failures/few', 'objectstore/bluestore-comp-zstd', 'openstack}', 'rados', 'rados/thrash-erasure-code-isa/{arch/x86_64', 'recovery-overrides/{more-async-recovery}', 'supported-random-distro$/{centos_8}', 'thrashers/none', 'thrashosds-health', 'workloads/ec-rados-plugin=isa-k=2-m=1}'] Failure: 1 jobs: ['7224308'] suites: ['clusters/{fixed-2', 'mon_election/connectivity', 'msgr-failures/many', 'msgr/async-v2only', 'objectstore/bluestore-comp-lz4', 'openstack}', 'rados', 'rados/basic/{ceph', 'supported-random-distro$/{centos_8}', 'tasks/scrub_test}']